diff --git a/.changeset/e2e-phase-5a-nightly-framework.md b/.changeset/e2e-phase-5a-nightly-framework.md new file mode 100644 index 00000000..c62aedb3 --- /dev/null +++ b/.changeset/e2e-phase-5a-nightly-framework.md @@ -0,0 +1,5 @@ +--- +"playground-cli": patch +--- + +E2E suite now has a `test-nightly-no-publish` matrix that runs only on the daily schedule (06:00 UTC) and `workflow_dispatch`. Adds two nightly-only cells: `nightly-mod-miss` (registry-miss path for unknown domains) and `nightly-diagnostic` (DOT_DEPLOY_VERBOSE / DOT_MEMORY_TRACE coverage). Per-PR runs are unaffected. diff --git a/.github/actions/setup-e2e/action.yml b/.github/actions/setup-e2e/action.yml new file mode 100644 index 00000000..88baf927 --- /dev/null +++ b/.github/actions/setup-e2e/action.yml @@ -0,0 +1,47 @@ +name: Setup E2E environment +description: Install pnpm/node/bun/kubo, run pnpm install, and resolve DOT_TAG from the trigger event. The CALLER must run actions/checkout BEFORE this action — local composite actions can't be located until the repo is on disk. + +outputs: + tag: + description: The DOT_TAG value resolved from github.event_name (e2e-ci-{nightly,dispatch,pr}). + value: ${{ steps.tag.outputs.tag }} + +runs: + using: composite + steps: + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install Kubo (IPFS) + shell: bash + run: | + wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz + tar xf kubo_v0.32.1_linux-amd64.tar.gz + sudo mv kubo/ipfs /usr/local/bin/ + ipfs init --profile=test + ipfs version + + - name: pnpm install + shell: bash + run: pnpm install + + - name: Resolve DOT_TAG from trigger + id: tag + shell: bash + run: | + # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch. + # Phase 7 will add `release` (with TAG=e2e-ci-release). + case "${{ github.event_name }}" in + schedule) TAG=e2e-ci-nightly ;; + workflow_dispatch) TAG=e2e-ci-dispatch ;; + *) TAG=e2e-ci-pr ;; + esac + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 49c9fe76..68a28e4a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -29,51 +29,27 @@ jobs: matrix: include: - cell: pr-install + # source: e2e/cli/install.test.ts → describe("dot install") pattern: "dot install" - cell: pr-preflight + # sources: e2e/cli/build.test.ts → describe("dot build") + # e2e/cli/deploy.test.ts → describe("dot deploy — preflight and validation") pattern: "dot build|preflight and validation" - cell: pr-mod + # source: e2e/cli/mod.test.ts → describe("dot mod — clone") pattern: "dot mod — clone" - cell: pr-init-session + # sources: e2e/cli/init.test.ts → describe("dot init …") + # e2e/cli/session.test.ts → describe("session management") pattern: "dot init|session management" outputs: - tag: ${{ steps.tag.outputs.tag }} + tag: ${{ steps.setup.outputs.tag }} steps: + # checkout must run before the local composite action can be loaded. - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - - uses: actions/setup-node@v4 - with: - node-version: "22" - - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Install Kubo (IPFS) - run: | - wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz - tar xf kubo_v0.32.1_linux-amd64.tar.gz - sudo mv kubo/ipfs /usr/local/bin/ - ipfs init --profile=test - ipfs version - - - run: pnpm install - - - name: Resolve DOT_TAG from trigger - id: tag - shell: bash - run: | - # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch. - # Phase 7 will add `release` (with TAG=e2e-ci-release). - case "${{ github.event_name }}" in - schedule) TAG=e2e-ci-nightly ;; - workflow_dispatch) TAG=e2e-ci-dispatch ;; - *) TAG=e2e-ci-pr ;; - esac - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})" + - id: setup + uses: ./.github/actions/setup-e2e - name: Run E2E cell (one retry on transient testnet failures) uses: nick-fields/retry@v3 @@ -81,12 +57,12 @@ jobs: timeout_minutes: 20 max_attempts: 2 retry_wait_seconds: 30 - command: pnpm exec vitest run --config e2e/vitest.config.ts -t "${{ matrix.pattern }}" + command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}" env: TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors DOT_DEPLOY_VERBOSE: "1" - DOT_TAG: ${{ steps.tag.outputs.tag }} + DOT_TAG: ${{ steps.setup.outputs.tag }} DOT_TELEMETRY: "1" - name: Upload forensic artefacts @@ -109,62 +85,83 @@ jobs: matrix: include: - cell: pr-deploy-frontend + # source: e2e/cli/deploy.test.ts → describe("dot deploy --playground — full pipeline …") pattern: "full pipeline" - cell: pr-deploy-foundry + # source: e2e/cli/deploy.test.ts → describe("dot deploy — foundry …") pattern: "deploy — foundry" # pr-deploy-cdm dropped pending fixture upgrade — see the # describe.skip block in e2e/cli/deploy.test.ts. Phase 5 # follow-up will land a real flipper fixture and re-add # this cell. steps: + # checkout must run before the local composite action can be loaded. - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - id: setup + uses: ./.github/actions/setup-e2e - - uses: actions/setup-node@v4 + - name: Run E2E cell (one retry on transient testnet failures) + uses: nick-fields/retry@v3 with: - node-version: "22" + timeout_minutes: 25 + max_attempts: 2 + retry_wait_seconds: 30 + command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}" + env: + TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot + TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors + DOT_DEPLOY_VERBOSE: "1" + DOT_TAG: ${{ steps.setup.outputs.tag }} + DOT_TELEMETRY: "1" - - uses: oven-sh/setup-bun@v2 + - name: Upload forensic artefacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 with: - bun-version: latest - - - name: Install Kubo (IPFS) - run: | - wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz - tar xf kubo_v0.32.1_linux-amd64.tar.gz - sudo mv kubo/ipfs /usr/local/bin/ - ipfs init --profile=test - ipfs version + name: e2e-reports-${{ matrix.cell }} + path: e2e-reports/ + retention-days: 7 + if-no-files-found: ignore - - run: pnpm install + test-nightly-no-publish: + name: "E2E · ${{ matrix.cell }}" + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: 5 + matrix: + include: + - cell: nightly-mod-miss + # source: e2e/cli/mod.test.ts → describe("dot mod — registry miss") + pattern: "dot mod — registry miss" + - cell: nightly-diagnostic + # source: e2e/cli/diagnostic.test.ts → describe("diagnostic mode") + # testFile scoping skips globalSetup chain-RPC calls (no chain access needed for diagnostic flag tests). + pattern: "diagnostic mode" + testFile: e2e/cli/diagnostic.test.ts + steps: + # checkout must run before the local composite action can be loaded. + - uses: actions/checkout@v4 - - name: Resolve DOT_TAG from trigger - id: tag - shell: bash - run: | - # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch. - # Phase 7 will add `release` (with TAG=e2e-ci-release). - case "${{ github.event_name }}" in - schedule) TAG=e2e-ci-nightly ;; - workflow_dispatch) TAG=e2e-ci-dispatch ;; - *) TAG=e2e-ci-pr ;; - esac - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})" + - id: setup + uses: ./.github/actions/setup-e2e - name: Run E2E cell (one retry on transient testnet failures) uses: nick-fields/retry@v3 with: - timeout_minutes: 25 + timeout_minutes: 20 max_attempts: 2 retry_wait_seconds: 30 - command: pnpm exec vitest run --config e2e/vitest.config.ts -t "${{ matrix.pattern }}" + command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}" env: TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors DOT_DEPLOY_VERBOSE: "1" - DOT_TAG: ${{ steps.tag.outputs.tag }} + DOT_TAG: ${{ steps.setup.outputs.tag }} DOT_TELEMETRY: "1" - name: Upload forensic artefacts @@ -179,12 +176,15 @@ jobs: report: name: E2E Report - needs: [test-no-publish, test-publish] + needs: [test-no-publish, test-publish, test-nightly-no-publish] if: always() runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Download JUnit + forensic artefacts + # Phase 1 has one upload (e2e-reports-current). When Phase 4 adds + # the matrix, drop `merge-multiple: true` and parse per-cell sub-dirs + # so per-leg junit.xml don't collide on the same flat path. uses: actions/download-artifact@v4 with: pattern: e2e-reports-* @@ -217,16 +217,22 @@ jobs: env: AGGREGATE_PUBLISH: ${{ needs.test-publish.result }} AGGREGATE_NO_PUBLISH: ${{ needs.test-no-publish.result }} + AGGREGATE_NIGHTLY_NO_PUBLISH: ${{ needs.test-nightly-no-publish.result }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} TAG: ${{ needs.test-no-publish.outputs.tag }} shell: /usr/bin/bash -euo pipefail {0} run: | - if [ "$AGGREGATE_PUBLISH" = "success" ] && [ "$AGGREGATE_NO_PUBLISH" = "success" ]; then - AGGREGATE=success - elif [ "$AGGREGATE_PUBLISH" = "cancelled" ] || [ "$AGGREGATE_NO_PUBLISH" = "cancelled" ]; then + # Gather results across all matrices. Add each new matrix job + # here when introducing one (Phase 5b+). 'skipped' (a matrix + # that didn't run because of an event-name gate) doesn't + # count as failure. + all_results="$AGGREGATE_PUBLISH $AGGREGATE_NO_PUBLISH $AGGREGATE_NIGHTLY_NO_PUBLISH" + if echo "$all_results" | grep -q failure; then + AGGREGATE=failure + elif echo "$all_results" | grep -q cancelled; then AGGREGATE=cancelled else - AGGREGATE=failure + AGGREGATE=success fi emoji() { case "$1" in @@ -358,7 +364,7 @@ jobs: - name: Open failure issue if: > (github.event_name == 'schedule' || github.event_name == 'release') && - (needs.test-no-publish.result != 'success' || needs.test-publish.result != 'success') + (needs.test-no-publish.result != 'success' || needs.test-publish.result != 'success' || needs.test-nightly-no-publish.result != 'success') env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.test-no-publish.outputs.tag }} diff --git a/CLAUDE.md b/CLAUDE.md index 2f72f25b..f7e225f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,7 @@ These are things that aren't self-evident from reading the code and have bitten - **Local launcher:** `tools/e2e-local.sh [smoke|pr|nightly]` — also callable via `pnpm test:e2e:smoke`, `pnpm test:e2e:pr`, `pnpm test:e2e:nightly`. - **CI workflow:** `.github/workflows/e2e.yml` — runs on PR / push:main / cron 06:00 UTC / workflow_dispatch. -- **CI matrix:** 7 cells in two matrices — `test-no-publish` (parallel: pr-install, pr-preflight, pr-mod, pr-init-session) + `test-publish` (max-parallel: 1: pr-deploy-frontend, pr-deploy-foundry, pr-deploy-cdm). Each cell runs a subset via `vitest -t ""`. +- **CI matrix:** 8 cells across three matrices — `test-no-publish` (parallel: pr-install, pr-preflight, pr-mod, pr-init-session) + `test-publish` (max-parallel: 1: pr-deploy-frontend, pr-deploy-foundry) + `test-nightly-no-publish` (parallel, schedule/dispatch only: nightly-mod-miss, nightly-diagnostic). Each cell runs a subset via `vitest -t ""`. - **Test files:** `e2e/cli/*.test.ts` (vitest, spawned via `bun run src/index.ts`). - **Reports directory:** `e2e-reports/junit.xml` + `e2e-reports/dot-runs.log` (gitignored). - **Tag prefix:** `DOT_TAG=e2e-{ci|local}-{trigger}` so Sentry dashboards filter test traffic. The CLI plumbs `DOT_TAG` into the `cli.tag` root-span attribute via `src/telemetry-config.ts`.