Skip to content

Commit 090544d

Browse files
EnderOfWorlds007EnderOfWorlds007
authored
Merge pull request #82 from paritytech/ci/e2e-phase-1-reporting
ci: E2E suite Phase 1 — reporting + tagging
2 parents ae7b208 + add5a34 commit 090544d

10 files changed

Lines changed: 365 additions & 14 deletions

File tree

‎.changeset/e2e-phase-1-reporting.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"playground-cli": patch
3+
---
4+
5+
CI now posts a sticky E2E test-pass comment on every PR with per-test pass/fail counts and Sentry triage links. Nightly schedule failures auto-open a GitHub issue. Per-cell forensic logs (`dot-runs.log`) and JUnit XML are uploaded as workflow artefacts. Test traffic is tagged with `cli.tag:e2e-ci-{pr|nightly|dispatch}` so production Sentry dashboards can filter it out.

‎.github/workflows/e2e.yml‎

Lines changed: 250 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,22 @@ on:
99
- cron: "0 6 * * *"
1010
workflow_dispatch:
1111

12+
permissions:
13+
pull-requests: write # sticky PR comment posting
14+
issues: write # auto-issue on schedule fail
15+
actions: read # /runs/{id}/jobs API for per-leg fetch
16+
17+
concurrency:
18+
group: e2e-${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
19+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
20+
1221
jobs:
13-
e2e:
14-
name: E2E Tests
22+
test:
23+
name: "E2E · current"
1524
runs-on: ubuntu-latest
16-
timeout-minutes: 30
25+
timeout-minutes: 55
26+
outputs:
27+
tag: ${{ steps.tag.outputs.tag }}
1728
steps:
1829
- uses: actions/checkout@v4
1930

@@ -37,11 +48,244 @@ jobs:
3748
3849
- run: pnpm install
3950

40-
- name: Run E2E tests
41-
run: pnpm test:e2e
51+
- name: Resolve DOT_TAG from trigger
52+
id: tag
53+
shell: bash
54+
run: |
55+
# Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch.
56+
# Phase 7 will add `release` (with TAG=e2e-ci-release).
57+
case "${{ github.event_name }}" in
58+
schedule) TAG=e2e-ci-nightly ;;
59+
workflow_dispatch) TAG=e2e-ci-dispatch ;;
60+
*) TAG=e2e-ci-pr ;;
61+
esac
62+
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
63+
echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})"
64+
65+
- name: Run E2E tests (one retry on transient testnet failures)
66+
uses: nick-fields/retry@v3
67+
with:
68+
timeout_minutes: 25
69+
max_attempts: 2
70+
retry_wait_seconds: 30
71+
command: pnpm test:e2e
4272
env:
4373
TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
4474
TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
4575
DOT_DEPLOY_VERBOSE: "1"
46-
DOT_TAG: e2e-ci
76+
DOT_TAG: ${{ steps.tag.outputs.tag }}
4777
DOT_TELEMETRY: "1"
78+
79+
- name: Upload forensic artefacts
80+
if: always()
81+
continue-on-error: true
82+
uses: actions/upload-artifact@v4
83+
with:
84+
name: e2e-reports-current
85+
path: e2e-reports/
86+
retention-days: 7
87+
if-no-files-found: ignore
88+
89+
report:
90+
name: E2E Report
91+
needs: [test]
92+
if: always()
93+
runs-on: ubuntu-latest
94+
timeout-minutes: 5
95+
steps:
96+
- name: Download JUnit + forensic artefacts
97+
# Phase 1 has one upload (e2e-reports-current). When Phase 4 adds
98+
# the matrix, drop `merge-multiple: true` and parse per-cell sub-dirs
99+
# so per-leg junit.xml don't collide on the same flat path.
100+
uses: actions/download-artifact@v4
101+
with:
102+
pattern: e2e-reports-*
103+
path: artefacts/
104+
merge-multiple: true
105+
106+
- name: Install xmlstarlet (for JUnit parsing)
107+
run: sudo apt-get update -q && sudo apt-get install -y -q xmlstarlet
108+
109+
- name: Fetch per-leg job conclusions
110+
id: legs
111+
env:
112+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
113+
shell: /usr/bin/bash -euo pipefail {0}
114+
run: |
115+
curl -fsSL \
116+
-H "Authorization: bearer $GITHUB_TOKEN" \
117+
-H "Accept: application/vnd.github+json" \
118+
"${{ github.api_url }}/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
119+
| jq -r '.jobs[] | select(.name | startswith("E2E · ")) | "\(.name)\t\(.conclusion)\t\((.completed_at // now | fromdateiso8601) - (.started_at // now | fromdateiso8601))"' \
120+
> /tmp/legs.tsv
121+
if [ ! -s /tmp/legs.tsv ]; then
122+
echo "::error::No 'E2E · …' jobs found in run ${{ github.run_id }}; matrix-job naming may have changed."
123+
exit 1
124+
fi
125+
echo "Per-leg conclusions:"
126+
cat /tmp/legs.tsv
127+
128+
- name: Parse JUnit and render report body
129+
env:
130+
AGGREGATE: ${{ needs.test.result }}
131+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
132+
TAG: ${{ needs.test.outputs.tag }}
133+
shell: /usr/bin/bash -euo pipefail {0}
134+
run: |
135+
emoji() { case "$1" in
136+
success) echo "✅ PASS";;
137+
failure) echo "❌ FAIL";;
138+
skipped) echo "⏭️ SKIP";;
139+
cancelled) echo "🚫 CANCEL";;
140+
*) echo "❓ $1";;
141+
esac; }
142+
143+
BRANCH="${{ github.head_ref || github.ref_name }}"
144+
SHA_SHORT="${GITHUB_SHA:0:7}"
145+
TAG_LABEL="${TAG:-e2e-ci-pr}"
146+
147+
# ---- Part 1: per-cell summary table ----
148+
{
149+
echo "<!-- e2e-pr-report -->"
150+
echo "## E2E Test Pass · $(emoji "$AGGREGATE")"
151+
echo ""
152+
echo "Tag: \`$TAG_LABEL\` · Branch: \`$BRANCH\` · Commit: \`$SHA_SHORT\` · [Run logs]($RUN_URL)"
153+
echo ""
154+
echo "| Cell | Result | Time |"
155+
echo "|------|--------|------|"
156+
while IFS=$'\t' read -r name conclusion duration; do
157+
dur_min=$(awk -v d="$duration" 'BEGIN{ printf "%dm%02ds", int(d/60), int(d%60) }')
158+
echo "| \`${name#E2E · }\` | $(emoji "$conclusion") | $dur_min |"
159+
done < /tmp/legs.tsv
160+
echo ""
161+
} > /tmp/body.md
162+
163+
# ---- Part 2: failures detail block (only if any test failed) ----
164+
FAILS_FILE=/tmp/failures.tsv
165+
: > "$FAILS_FILE"
166+
167+
# nullglob so non-matching globs expand to nothing.
168+
shopt -s nullglob
169+
XML_FILES=( artefacts/junit*.xml artefacts/*/junit*.xml )
170+
shopt -u nullglob
171+
172+
if command -v xmlstarlet >/dev/null 2>&1; then
173+
for xml in "${XML_FILES[@]}"; do
174+
xmlstarlet sel -t -m "//testcase[failure or error]" \
175+
-v "concat(@classname, ' › ', @name)" -o $'\t' \
176+
-v "failure/@message | error/@message" -n "$xml" \
177+
>> "$FAILS_FILE" || true
178+
done
179+
else
180+
for xml in "${XML_FILES[@]}"; do
181+
grep -E '<testcase|<failure|<error' "$xml" \
182+
| awk '/<testcase/{name=$0} /<failure|<error/{print name "\t" $0}' \
183+
>> "$FAILS_FILE" || true
184+
done
185+
fi
186+
187+
if [ -s "$FAILS_FILE" ]; then
188+
FAIL_COUNT=$(wc -l < "$FAILS_FILE")
189+
MAX_LINES=100
190+
TRUNCATED=$([ "$FAIL_COUNT" -gt "$MAX_LINES" ] && echo "1" || echo "0")
191+
{
192+
echo "<details><summary>❌ Failed tests ($FAIL_COUNT)</summary>"
193+
echo ""
194+
head -n "$MAX_LINES" "$FAILS_FILE" | while IFS=$'\t' read -r name msg; do
195+
msg_clean=$(echo "$msg" | sed -E 's/<[^>]+>//g; s/&quot;/"/g; s/&lt;/</g; s/&gt;/>/g; s/&amp;/\&/g')
196+
msg_first3=$(echo "$msg_clean" | head -3 | sed 's/^/ /')
197+
echo "- \`$name\`"
198+
echo "$msg_first3"
199+
done
200+
if [ "$TRUNCATED" = "1" ]; then
201+
REMAINING=$((FAIL_COUNT - MAX_LINES))
202+
echo ""
203+
echo "... and $REMAINING more failures (see [run logs]($RUN_URL))"
204+
fi
205+
echo ""
206+
echo "</details>"
207+
echo ""
208+
} >> /tmp/body.md
209+
fi
210+
211+
# ---- Part 3: Sentry traces link ----
212+
# Project ID 4511298552135760 mirrors src/telemetry-config.ts (PLAYGROUND_SENTRY_DSN
213+
# → o4511059872841728 / 4511298552135760) and sentry/dashboards/2143100.json.
214+
# If the project is ever rotated, update all three together.
215+
# Only the traces URL is emitted — it works against any cli.tag value.
216+
# The "E2E Health dashboard" link is intentionally NOT emitted in
217+
# Phase 1 (requires manual one-time dashboard creation per spec §9a).
218+
{
219+
echo "**Sentry traces:** [view spans for this run](https://paritytech.sentry.io/explore/traces/?project=4511298552135760&query=cli.tag%3A$TAG_LABEL)"
220+
} >> /tmp/body.md
221+
222+
echo "Rendered body:"
223+
cat /tmp/body.md
224+
echo ""
225+
echo "---- writing to step summary ----"
226+
cat /tmp/body.md >> "$GITHUB_STEP_SUMMARY"
227+
228+
- name: Post or update PR comment
229+
if: github.event_name == 'pull_request'
230+
uses: actions/github-script@v7
231+
with:
232+
script: |
233+
const fs = require('fs');
234+
const body = fs.readFileSync('/tmp/body.md', 'utf8');
235+
const marker = '<!-- e2e-pr-report -->';
236+
const prNumber = context.issue.number;
237+
const { data: comments } = await github.rest.issues.listComments({
238+
owner: context.repo.owner,
239+
repo: context.repo.repo,
240+
issue_number: prNumber,
241+
});
242+
const existing = comments.find(c => c.body && c.body.includes(marker));
243+
if (existing) {
244+
await github.rest.issues.updateComment({
245+
owner: context.repo.owner,
246+
repo: context.repo.repo,
247+
comment_id: existing.id,
248+
body,
249+
});
250+
console.log(`Updated existing comment ${existing.id}`);
251+
} else {
252+
await github.rest.issues.createComment({
253+
owner: context.repo.owner,
254+
repo: context.repo.repo,
255+
issue_number: prNumber,
256+
body,
257+
});
258+
console.log("Created new comment");
259+
}
260+
261+
- name: Open failure issue
262+
if: >
263+
(github.event_name == 'schedule' || github.event_name == 'release') &&
264+
needs.test.result != 'success'
265+
env:
266+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
267+
TAG: ${{ needs.test.outputs.tag }}
268+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
269+
API_URL: ${{ github.api_url }}
270+
REPO: ${{ github.repository }}
271+
shell: /usr/bin/bash -euo pipefail {0}
272+
run: |
273+
TODAY=$(date -u +%Y-%m-%d)
274+
TITLE="Nightly E2E failure: $TODAY"
275+
CONTEXT="Nightly E2E failed on $TODAY (tag \`$TAG\`)."
276+
# Release-trigger branches will be added in Phase 7 (post-Phase-4).
277+
278+
{
279+
echo "$CONTEXT"
280+
echo ""
281+
cat /tmp/body.md
282+
echo ""
283+
echo "- Run: $RUN_URL"
284+
} > /tmp/issue-body.md
285+
286+
BODY=$(cat /tmp/issue-body.md)
287+
curl -fsSL -X POST \
288+
-H "Authorization: bearer $GITHUB_TOKEN" \
289+
-H "Accept: application/vnd.github+json" \
290+
"$API_URL/repos/$REPO/issues" \
291+
-d "$(jq -n --arg title "$TITLE" --arg body "$BODY" '{title: $title, body: $body}')"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ docs-internal/
66
.worktrees/
77
reference-repos/
88
.tokensave
9+
e2e-reports/

‎CLAUDE.md‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,16 @@ These are things that aren't self-evident from reading the code and have bitten
4949
- `2216067.json` — **Playground CLI Failures** (per-error-type drill-downs).
5050
- `2216096.json` — **Playground CLI E2E Health** (inverse filter, `cli.tag:e2e-*`).
5151
- **Workflow:** run `./sentry/backup-dashboards.sh` BEFORE any change. Use `./sentry/patch-dashboard.py <id> <patch.json>` for surgical edits (supports `replace`, `patch_query`, `set_description` ops) or full widget replacement. Use `./sentry/create-dashboard.py <payload.json>` for new dashboards. Per spec §15f, do NOT include a `projects` field in POST payloads. Per spec §15g, PUT replaces the whole widget list — backup first.
52-
- **E2E tagging:** every spawn from `e2e/cli/helpers/dot.ts` injects `DOT_TAG=e2e-local` (default) and `DOT_TELEMETRY=1`. CI sets `DOT_TAG=e2e-ci` in `.github/workflows/e2e.yml` so production health widgets filter cleanly via `!cli.tag:e2e-*`.
52+
- **E2E tagging:** every spawn from `e2e/cli/helpers/dot.ts` injects `DOT_TAG=e2e-local` (fallback) and `DOT_TELEMETRY=1`. `tools/e2e-local.sh` overrides that to `e2e-local-{smoke|pr|nightly}` based on the mode argument. CI sets `DOT_TAG=e2e-ci-{pr|nightly|dispatch}` in `.github/workflows/e2e.yml` so production health widgets filter cleanly via `!cli.tag:e2e-*`.
5353
- **SAD% propagation** is verified by a regression test in `src/telemetry.test.ts` ("SAD% propagation through transaction envelope"). It confirms `captureWarning` flips `cli.sad="true"` on the root transaction. If that test fails, the SAD% dashboard widget on Dashboard 1 will silently degrade to a duplicate of the unexpected-failure rate.
54+
55+
## E2E Tests
56+
57+
- **Local launcher:** `tools/e2e-local.sh [smoke|pr|nightly]` — also callable via `pnpm test:e2e:smoke`, `pnpm test:e2e:pr`, `pnpm test:e2e:nightly`.
58+
- **CI workflow:** `.github/workflows/e2e.yml` — runs on PR / push:main / cron 06:00 UTC / workflow_dispatch.
59+
- **Test files:** `e2e/cli/*.test.ts` (vitest, spawned via `bun run src/index.ts`).
60+
- **Reports directory:** `e2e-reports/junit.xml` + `e2e-reports/dot-runs.log` (gitignored).
61+
- **Tag prefix:** `DOT_TAG=e2e-{ci|local}-{trigger}` so Sentry dashboards filter test traffic. The CLI plumbs `DOT_TAG` into the `cli.tag` root-span attribute via `src/telemetry-config.ts`.
62+
- **CI report job name:** `E2E Report` — aggregates per-leg conclusions, posts a sticky PR comment with marker `<!-- e2e-pr-report -->`, opens an auto-issue on schedule/release fail.
63+
- **Bootstrap:** see `tools/register-mod-fixture.ts` for the mod-test fixture; full bootstrap doc TBD in a later phase.
64+
- **Design spec:** `docs-internal/2026-05-02-e2e-test-suite-design.md`.

‎e2e/cli/helpers/dot.ts‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77

88
import { execa as execaFn } from "execa";
9+
import { appendFileSync, existsSync } from "node:fs";
910
import { resolve } from "node:path";
1011

1112
const REPO_ROOT = resolve(import.meta.dirname, "../../../");
@@ -74,20 +75,47 @@ async function run(args: string[], options?: DotOptions): Promise<DotResult> {
7475
env.HOME = options.home;
7576
}
7677

78+
const startedAt = Date.now();
7779
try {
7880
const result = await execaFn("bun", ["run", CLI_ENTRY, ...args], {
7981
cwd: options?.cwd ?? REPO_ROOT,
8082
env,
8183
timeout: options?.timeout ?? DEFAULT_TIMEOUT,
8284
reject: false,
8385
});
84-
return {
86+
const dotResult: DotResult = {
8587
stdout: result.stdout,
8688
stderr: result.stderr,
8789
exitCode: result.exitCode ?? 1,
8890
};
91+
appendForensicLog(args, dotResult, Date.now() - startedAt);
92+
return dotResult;
8993
} catch (err: unknown) {
9094
const msg = err instanceof Error ? err.message : String(err);
91-
return { stdout: "", stderr: msg, exitCode: 1 };
95+
const dotResult: DotResult = { stdout: "", stderr: msg, exitCode: 1 };
96+
appendForensicLog(args, dotResult, Date.now() - startedAt);
97+
return dotResult;
98+
}
99+
}
100+
101+
function appendForensicLog(args: string[], result: DotResult, durationMs: number): void {
102+
// Best-effort: only write if e2e-reports/ exists. Don't create the dir
103+
// eagerly — tests that don't care about forensic capture shouldn't get
104+
// a stray dir in their cwd.
105+
const reportsDir = resolve(REPO_ROOT, "e2e-reports");
106+
if (!existsSync(reportsDir)) return;
107+
try {
108+
const entry = [
109+
`# ${new Date().toISOString()} exit=${result.exitCode} durationMs=${durationMs}`,
110+
`# args: ${args.map((a) => (/\s/.test(a) ? `'${a}'` : a)).join(" ")}`,
111+
`--- stdout ---`,
112+
result.stdout,
113+
`--- stderr ---`,
114+
result.stderr,
115+
``,
116+
].join("\n");
117+
appendFileSync(resolve(reportsDir, "dot-runs.log"), entry);
118+
} catch {
119+
// Forensic logging must never fail a test. Swallow.
92120
}
93121
}

‎e2e/vitest.config.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { defineConfig } from "vitest/config";
22

3+
const isCI = process.env.CI === "true";
4+
35
export default defineConfig({
46
test: {
57
include: ["e2e/**/*.test.ts"],
@@ -10,5 +12,10 @@ export default defineConfig({
1012
// Chain client WebSockets may keep the event loop alive after teardown.
1113
// Force exit after tests complete rather than hanging.
1214
teardownTimeout: 5_000,
15+
// Always emit JUnit XML for the report job. Add a human-readable
16+
// streaming reporter on local runs so developers see live progress;
17+
// CI runs strip it because the run logs are noisy enough already.
18+
reporters: isCI ? ["junit"] : ["default", "junit"],
19+
outputFile: { junit: "e2e-reports/junit.xml" },
1320
},
1421
});

‎package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
"cli:install": "bun build --compile src/index.ts --outfile ~/.polkadot/bin/dot",
1717
"test": "vitest run",
1818
"test:watch": "vitest",
19-
"test:e2e": "vitest run --config e2e/vitest.config.ts"
19+
"test:e2e": "vitest run --config e2e/vitest.config.ts",
20+
"test:e2e:smoke": "tools/e2e-local.sh smoke",
21+
"test:e2e:pr": "tools/e2e-local.sh pr",
22+
"test:e2e:nightly": "tools/e2e-local.sh nightly"
2023
},
2124
"dependencies": {
2225
"@dotdm/contracts": "latest",

0 commit comments

Comments
 (0)