Skip to content

Commit 1e2e21e

Browse files
author
AztecBot
committed
chore: sync public-v5-next with upstream v5-next
2 parents 8155b94 + 5239f8c commit 1e2e21e

47 files changed

Lines changed: 2771 additions & 1625 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

ci.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ function print_usage {
4242
echo_cmd "shell" "Drop into a shell in the current running build instance container."
4343
echo_cmd "shell-host" "Drop into a shell in the current running build host."
4444
echo_cmd "log" "Display the log of the given log ID."
45+
echo_cmd "test-timings" "Download per-test timing JSONL for a job: test-timings <ci_log_id> <folder>."
4546
echo_cmd "kill" "Terminate running EC2 instance with instance_name."
4647
echo_cmd "draft" "Mark the current PR as draft (no automatic CI runs when pushing)."
4748
echo_cmd "ready" "Mark the current PR as ready (enable automatic CI runs when pushing)."
@@ -424,6 +425,29 @@ case "$cmd" in
424425
fi
425426
;;
426427

428+
test-timings)
429+
# Download all per-test timing files for a CI job and gunzip them into a folder.
430+
# ci_log_id is the job's top-level log id (the decimal id in its ci.aztec-labs.com URL).
431+
# Each downloaded file is named after the test's individual log id (ci.aztec-labs.com/<log_id>).
432+
# Usage: ./ci.sh test-timings <ci_log_id> <folder>
433+
ci_log_id="${1:-}"
434+
folder="${2:-}"
435+
if [ -z "$ci_log_id" ] || [ -z "$folder" ]; then
436+
echo "usage: $(basename $0) test-timings <ci_log_id> <folder>"
437+
exit 1
438+
fi
439+
mkdir -p "$folder"
440+
aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp --recursive \
441+
"s3://aztec-ci-artifacts/logs/test-timings/${ci_log_id}/" "$folder/"
442+
for f in "$folder"/*.log.gz; do
443+
[ -e "$f" ] || continue
444+
out="${f%.log.gz}.jsonl"
445+
gunzip -c "$f" > "$out"
446+
rm -f "$f"
447+
done
448+
echo "Downloaded test timings for job $ci_log_id into $folder/"
449+
;;
450+
427451
#################
428452
# PR MANAGEMENT #
429453
#################

ci3/docker_isolate

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ cid=$(docker run -d \
7676
-e USE_HOME_TMP \
7777
-e AVM \
7878
-e ELU_MONITOR_FILE \
79+
-e TEST_TIMING_FILE \
80+
-e RUN_ID \
7981
"${arg_env_vars[@]}" \
8082
aztecprotocol/build:3.0 \
8183
/bin/bash -c "$cmd")

ci3/exec_test

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ export VERBOSE=1
4646
# ELU monitor file path (e2e tests write event loop data here, uploaded after test finishes).
4747
export ELU_MONITOR_FILE=$HOME/.elu_monitor_$$.log
4848

49+
# Per-test timing file (e2e tests write JSONL here, uploaded to S3 after test finishes).
50+
# $$ is this exec_test pid, so it is unique per parallel test command.
51+
export TEST_TIMING_FILE=$HOME/.test_timing_$$.log
52+
4953
# Run the test with timestamps via process substitution (preserves exit code)
5054
set +e
5155
if [ "${ISOLATE:-0}" -eq 1 ]; then
@@ -64,4 +68,19 @@ if [ -s "$ELU_MONITOR_FILE" ]; then
6468
rm -f "$ELU_MONITOR_FILE"
6569
fi
6670

71+
# Upload per-test timing data to S3 if present (written by e2e tests, absent for C++/Rust/Noir).
72+
# Keyed by the job's CI_LOG_ID (the decimal id in a job's dashboard URL) and the individual test's
73+
# log id (LOG_ID, passed by run_test_cmd; same id as the test's log at ci.aztec-labs.com/<id>),
74+
# so a test log maps directly to its timing file.
75+
# Object lands at s3://aztec-ci-artifacts/logs/test-timings/<CI_LOG_ID>/<LOG_ID>.log.gz (gzipped JSONL).
76+
if [ -s "$TEST_TIMING_FILE" ]; then
77+
if [ -z "${CI_LOG_ID:-}" ] || [ -z "${LOG_ID:-}" ]; then
78+
echo "Warning: not uploading test timings; CI_LOG_ID or LOG_ID missing (CI_LOG_ID='${CI_LOG_ID:-}' LOG_ID='${LOG_ID:-}')."
79+
else
80+
echo "Test timing file: $TEST_TIMING_FILE ($(wc -l <"$TEST_TIMING_FILE") lines, $(wc -c <"$TEST_TIMING_FILE") bytes)"
81+
cat "$TEST_TIMING_FILE" | gzip | cache_s3_transfer_to test-timings "${CI_LOG_ID}/${LOG_ID}"
82+
fi
83+
rm -f "$TEST_TIMING_FILE"
84+
fi
85+
6786
exit $code

ci3/run_test_cmd

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,9 @@ function run_test {
199199
# Reset timer and run the test in background (for prompt signal handling) using exec_test.
200200
SECONDS=0
201201
set +e
202-
$ci3/exec_test "$cmd" >> "$tmp_file" 2>&1 &
202+
# Pass this attempt's log id so exec_test can name the test's timing file after it
203+
# (matching the test's log at ci.aztec-labs.com/<log_key>). Tracks rotate_log on retries.
204+
LOG_ID=$log_key $ci3/exec_test "$cmd" >> "$tmp_file" 2>&1 &
203205
test_pid=$!
204206
wait $test_pid
205207
code=$?

docs/docs-operate/operators/reference/cli-reference.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ tags:
3535
--network <value> ($NETWORK)
3636
Network to run Aztec on
3737

38-
--enable-version-check (default: true) ($ENABLE_VERSION_CHECK)
39-
Check if the node is running the latest version and is following the latest rollup
38+
--enable-auto-shutdown (default: false) ($ENABLE_AUTO_SHUTDOWN)
39+
Soft-shutdown the node when the canonical rollup is no longer compatible (protocol constants diverge), keeping the health server up so K8s probes keep passing. Only applies to nodes following the canonical rollup.
4040

4141
--sync-mode <value> (default: snapshot) ($SYNC_MODE)
4242
Set sync mode to `full` to always sync via L1, `snapshot` to download a snapshot if there is no local data, `force-snapshot` to download even if there is local data.

spartan/environments/network-defaults.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ networks:
264264
SLASH_INVALID_BLOCK_PENALTY: 10e18
265265
SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0
266266
SLASH_GRACE_PERIOD_L2_SLOTS: 0
267-
ENABLE_VERSION_CHECK: true
267+
ENABLE_AUTO_SHUTDOWN: false
268268

269269
testnet:
270270
<<: *prodlike
@@ -310,7 +310,7 @@ networks:
310310
SLASH_INVALID_BLOCK_PENALTY: 100000e18
311311
SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 100000e18 # AZIP-16: activated at SMALL
312312
SLASH_GRACE_PERIOD_L2_SLOTS: 64
313-
ENABLE_VERSION_CHECK: true
313+
ENABLE_AUTO_SHUTDOWN: false
314314

315315
mainnet:
316316
<<: *prodlike
@@ -355,7 +355,7 @@ networks:
355355
# Telemetry
356356
PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ""
357357
PUBLIC_OTEL_COLLECT_FROM: ""
358-
ENABLE_VERSION_CHECK: false
358+
ENABLE_AUTO_SHUTDOWN: false
359359
# Slasher penalties - more lenient initially
360360
SLASH_DATA_WITHHOLDING_PENALTY: 0
361361
SLASH_INACTIVITY_TARGET_PERCENTAGE: 0.8
Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
---
2+
name: monitor-pr
3+
description: Monitor a PR on a ~10-minute loop and drive it to green and conflict-free — watch CI, dispatch fixers for failures (new commits, never amend), and resolve base conflicts (rebase by default, merge if the branch already merged the base). Stop and report when the PR is green with no conflicts.
4+
argument-hint: <PR number>
5+
---
6+
7+
# Monitor PR
8+
9+
A self-paced monitoring loop over a single PR. Each iteration takes a status snapshot, acts on it
10+
(fix CI failures, resolve conflicts, or do nothing while CI is pending), then re-schedules itself
11+
~10 minutes out. The loop ends — and reports success — only when the PR is green AND conflict-free.
12+
13+
This skill **composes** its siblings rather than re-deriving them:
14+
- Fix CI failures the way `/fix-pr` does (delegating identification to the `identify-ci-failures`
15+
subagent, then root-causing the failure — preferring a real fix over a skip or flake flag).
16+
- Resolve conflicts the way `/rebase-pr` does.
17+
18+
## Usage
19+
20+
```
21+
/monitor-pr 19882
22+
```
23+
24+
If no PR number is given, the loop targets the PR for the currently checked-out branch.
25+
26+
## Hard rules (read first)
27+
28+
- **New commits only — NEVER `git commit --amend`.** This loop runs unattended and re-pushes
29+
repeatedly; amending would rewrite work that may already be in flight. This overrides the
30+
amend-for-`next` guidance in `/fix-pr` and `/rebase-pr` — in *this* skill, every fix and every
31+
conflict resolution is a fresh commit.
32+
- **Determine the base from the PR, never assume `master`/`main`.** The `baseRefName` from
33+
`gh pr view` is authoritative.
34+
- **Never `git add -A`/`-u`/`.`.** Stage named files only (especially when dispatching a fixer
35+
agent, which lacks this session's context for judging which changes belong to the task).
36+
- **Conventional-commit messages** (`fix:`, `refactor:`, …). **No `Co-Authored-By: Claude` trailer**
37+
and no "Generated with Claude Code" footer — the git author is the author of record.
38+
- **Bounded — the loop must be able to give up.** This runs unattended; a failure the fixer cannot
39+
resolve would otherwise re-dispatch a fixer every iteration forever, churning commits and tokens.
40+
Track progress across iterations in a state file and **stop + escalate to the human** when stuck
41+
(see "Step 0" and "Giving up" below). Never loop indefinitely on a failure that is not improving.
42+
43+
## Iteration workflow
44+
45+
### Step 0 — Load loop state
46+
47+
Each firing of `/monitor-pr <PR>` is a **cold turn** — nothing carries over from the previous
48+
iteration except what you persist. To detect lack of progress (and stop instead of looping forever),
49+
keep a small JSON state file across iterations, in your session scratchpad dir, named
50+
`monitor-pr-<PR>.json`:
51+
52+
```json
53+
{ "iterations": 0, "lastHeadSha": "", "fixAttempts": { "<check name>": 0 } }
54+
```
55+
56+
At the start of every iteration: read it (treat a missing file as the zero state above) and increment
57+
`iterations`. You will update `lastHeadSha` and `fixAttempts` in Step 2, and write the file back
58+
before rescheduling in Step 3.
59+
60+
### Step 1 — Snapshot the PR status
61+
62+
Run the bundled status script from inside the repo clone (it shells out to `gh`, `git`, `jq`):
63+
64+
```bash
65+
bash "$CLAUDE_SKILL_DIR/check-pr.sh" <PR> # or omit <PR> to use the current branch's PR
66+
```
67+
68+
It prints parseable `KEY=VALUE` lines:
69+
70+
- `STATE` — abort the loop if not `OPEN` (report: "PR #N is <state>, nothing to monitor.").
71+
- `BASE`, `HEAD` — base and head branch names (base is authoritative; use it everywhere below).
72+
- `MERGEABLE` / `MERGE_STATE``CONFLICTING` or `MERGE_STATE=DIRTY` means there are conflicts.
73+
- `HAS_BASE_MERGE_COMMITS``yes` if the branch already contains merge commit(s) from the base
74+
(`git log --merges origin/<base>..HEAD` is non-empty). This drives the rebase-vs-merge choice.
75+
- `CI``PENDING` (some checks still running), `FAIL` (≥1 check failed/cancelled), `PASS`
76+
(all finished and green), or `NONE`.
77+
- When `CI=FAIL`, a `FAILED_CHECKS_BEGIN … FAILED_CHECKS_END` block lists each failing check as
78+
`name<TAB>bucket<TAB>link` — capture these to hand to the fixer.
79+
80+
If the script can't resolve the PR or find CI logs, ask the user for the PR number / CI URL /
81+
CI password rather than guessing.
82+
83+
### Step 2 — Decide and act on the snapshot
84+
85+
Pick exactly one branch based on the snapshot, in this priority order:
86+
87+
**A. CI still `PENDING`** → do nothing this round. Go to Step 3 and reschedule.
88+
89+
**B. CI `FAIL`** → before dispatching, run two guards against the state file:
90+
91+
- **Stale-CI guard.** If the failing run is for an *older* commit than the current `origin/<head>`
92+
tip (compare the run's head SHA — from the check's run, or just `git rev-parse origin/<head>`
93+
against `lastHeadSha`), a previous fixer's push has not re-triggered CI yet. Treat this as
94+
`PENDING` (case A): do nothing, reschedule. This prevents dispatching a second fixer for a
95+
failure that is already being re-tested.
96+
- **Give-up guard.** For each currently-failing check, look at `fixAttempts[<check name>]`. If a
97+
check has already had **2** fix attempts and is still failing on a *newer* head SHA than when it
98+
was last attempted, the loop is not making progress on it — **stop and escalate** (see "Giving
99+
up").
100+
101+
Otherwise dispatch a **new fixer agent** (see "Dispatching the fixer" below). After it pushes,
102+
increment `fixAttempts[<check name>]` for each check it addressed and set `lastHeadSha` to the new
103+
`origin/<head>` tip. Wait for it to finish (it commits as new commits and pushes, which restarts
104+
CI), then go to Step 3. Do not also fix in this session — the fixer owns the edits.
105+
106+
**C. Conflicts present** (`MERGEABLE=CONFLICTING` or `MERGE_STATE=DIRTY`), CI not failing →
107+
resolve them yourself the way `/rebase-pr` does, choosing rebase vs. merge by the rule below, then
108+
push and go to Step 3.
109+
110+
**D. CI `PASS` and no conflicts****stop.** Do not reschedule. Report success (Step 4).
111+
112+
If both CI failed *and* conflicts exist, handle the conflicts first (B-style fixes won't land
113+
cleanly on a conflicted branch), then let the next iteration pick up the CI failures.
114+
115+
### Dispatching the fixer (case B)
116+
117+
Spawn one agent (Task / Agent tool) to fix the failures. It must receive **full context** so it
118+
never has to rediscover the failures:
119+
120+
- PR number, head branch, base branch.
121+
- Each failing check: its **name**, **bucket**, and **log link/hash** from the `FAILED_CHECKS`
122+
block.
123+
- The repro command for affected tests, and the instruction to pull logs the way `/fix-pr` does
124+
(via the `identify-ci-failures` subagent) and root-cause real test failures — prefer a genuine
125+
fix over a skip/flake flag; only re-run a job when a failure is clearly infra/flaky.
126+
127+
Bake these instructions into the fixer's prompt:
128+
- Check out the PR branch, verify locally (`yarn build` + affected tests) before pushing.
129+
- **Commit fixes as NEW commits — never `--amend`.** Stage named files only. Conventional-commit
130+
message, no Claude co-author trailer. Then `git push` (plain push; no force needed for new
131+
commits).
132+
- Report back what it changed and whether the push succeeded.
133+
134+
### Resolving conflicts (case C) — rebase vs. merge
135+
136+
Default to **rebase**, matching `/rebase-pr`. The one exception: if `HAS_BASE_MERGE_COMMITS=yes`,
137+
the branch already merged the base in, so **merge** the base instead (rebasing a branch with merge
138+
commits is error-prone and would rewrite that history).
139+
140+
```bash
141+
git fetch origin <base>
142+
gh pr checkout <PR> # if not already on the branch
143+
```
144+
145+
- **Rebase path** (`HAS_BASE_MERGE_COMMITS=no`):
146+
`git rebase origin/<base>` → resolve each conflict, `git add <named-files>`,
147+
`git rebase --continue`, repeat → verify (`yarn build`; bootstrap first if there are changes
148+
outside `yarn-project`, as `/rebase-pr` describes) → `git push --force-with-lease`.
149+
- **Merge path** (`HAS_BASE_MERGE_COMMITS=yes`):
150+
`git merge origin/<base>` → resolve each conflict, `git add <named-files>`,
151+
`git commit` (a new merge commit; never amend) → verify → `git push` (plain push).
152+
153+
Either way: stage named files only, and never amend existing commits.
154+
155+
### Giving up (stop without success)
156+
157+
The loop is meant to be unattended, so it must recognize when it is stuck and hand back to the human
158+
rather than spinning. **Stop and report (do NOT reschedule)** when any of these hold:
159+
160+
- **Per-check stall.** A failing check has reached **2** fix attempts (`fixAttempts[<name>] >= 2`)
161+
and still fails on a newer head SHA than the last attempt — the fixer cannot resolve it.
162+
- **Global cap.** `iterations` reaches **12** (~2 hours at the 10-minute cadence) without reaching
163+
green-and-conflict-free.
164+
165+
When giving up, report like Step 4 but framed as *unresolved*: PR number, what is still failing (the
166+
stuck check names + their log links) or still conflicting, how many iterations/fix attempts were
167+
spent, and a one-line ask for the human to take over. Then **omit `ScheduleWakeup`** and `TaskStop`
168+
any Monitor you armed. Do not delete the state file — it is the record of what was tried.
169+
170+
### Step 3 — Reschedule (the ~10-minute loop)
171+
172+
This skill self-paces using the same dynamic-loop contract as the `loop` skill. As the **last
173+
action of the turn** (after writing a one-line status of what you observed and did):
174+
175+
0. **Persist the state file** (`monitor-pr-<PR>.json`) with this iteration's updated `iterations`,
176+
`lastHeadSha`, and `fixAttempts` — otherwise the next cold turn cannot tell whether progress is
177+
being made and the give-up guards never fire.
178+
1. Optionally arm a **Monitor** (`persistent: true`) as the primary wake signal — e.g. poll
179+
`gh pr checks <PR>` and emit a line when any check leaves `pending`, or when the run finishes.
180+
Arm it once; on later iterations call `TaskList` first and skip if it's already running. With a
181+
Monitor armed, `ScheduleWakeup` becomes a fallback heartbeat rather than the only trigger.
182+
2. Call **`ScheduleWakeup`** with:
183+
- `delaySeconds: 600` (~10 minutes; lean longer, 1200–1800s, if a Monitor is the real trigger).
184+
- `reason`: one sentence on why (e.g. "next CI poll for PR #N").
185+
- `prompt: "/monitor-pr <PR>"` — verbatim, so the next firing re-enters this skill and continues
186+
the loop.
187+
188+
If woken by a `<task-notification>` from the Monitor instead of the scheduled prompt, handle the
189+
event (re-run Step 1–2 for that PR), then call `ScheduleWakeup` again with the same prompt and delay
190+
to reset the safety net.
191+
192+
### Step 4 — Stop and report (case D)
193+
194+
When the PR is green and conflict-free:
195+
- **Omit the `ScheduleWakeup` call** (this ends the loop) and `TaskStop` any Monitor you armed
196+
(use `TaskList` to find its ID if it's no longer in context).
197+
- Report: PR number, final CI status (all checks green), that there are no conflicts, and a short
198+
summary of anything the loop fixed or merged along the way.
199+
200+
## Key Points
201+
202+
- **Loop until green AND conflict-free**, then stop — green alone is not done if the PR still
203+
conflicts, and vice versa.
204+
- **Bounded — give up when stuck.** Persist `monitor-pr-<PR>.json` across cold turns; stop and
205+
escalate after 2 failed fix attempts on the same check or 12 iterations. Never loop forever.
206+
- **Self-pace via `ScheduleWakeup`** every ~10 min (`prompt: /monitor-pr <PR>`); a `persistent`
207+
Monitor on `gh pr checks` is the idiomatic faster wake signal. Stopping = simply not rescheduling.
208+
- **New commits only, never amend** — for both the dispatched fixer and your own conflict commits.
209+
- **Fixer gets full context** — PR/branches/base, every failing check name + log link, repro
210+
command — so it never re-investigates from scratch.
211+
- **Rebase by default; merge when the branch already merged the base** (`HAS_BASE_MERGE_COMMITS=yes`).
212+
- **Base comes from the PR, not assumptions.** Named `git add` only. Conventional commits, no Claude
213+
trailer.
214+
215+
## Reference
216+
217+
- `check-pr.sh` (this directory) — the status-snapshot script.
218+
- `/fix-pr` — CI-failure identification and fixing methodology.
219+
- `/rebase-pr` — conflict resolution and post-rebase verification.
220+
- `CLAUDE.md` — base-branch routing and repo guardrails.

0 commit comments

Comments
 (0)