Skip to content

Commit d3a19d6

Browse files
authored
Merge pull request #88 from clkao/spacedock-ensign/runtime-live-e2e-pr-trigger-and-environment-gate
PR-triggered runtime live E2E with environment approval gate
2 parents 110d9b9 + 7256f87 commit d3a19d6

13 files changed

Lines changed: 507 additions & 65 deletions

.github/workflows/runtime-live-e2e.yml

Lines changed: 79 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: Runtime Live E2E
22

33
on:
4+
pull_request:
45
workflow_dispatch:
56
inputs:
67
pr_number:
@@ -15,9 +16,15 @@ permissions:
1516
jobs:
1617
claude-live:
1718
runs-on: ubuntu-latest
19+
environment:
20+
name: CI-E2E
21+
deployment: false
1822
env:
1923
DISABLE_AUTOUPDATER: "1"
20-
PR_NUMBER: ${{ inputs.pr_number }}
24+
KEEP_TEST_DIR: "1"
25+
TRIGGER_SOURCE: ${{ github.event_name }}
26+
PR_NUMBER: ${{ github.event.pull_request.number }}
27+
DISPATCH_PR_NUMBER: ${{ inputs.pr_number }}
2128
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
2229
steps:
2330
- name: Check required secret
@@ -31,9 +38,21 @@ jobs:
3138
uses: actions/github-script@v7
3239
with:
3340
script: |
34-
const prNumber = Number(process.env.PR_NUMBER);
41+
const triggerSource = process.env.TRIGGER_SOURCE;
42+
let rawPrNumber;
43+
44+
if (triggerSource === "pull_request") {
45+
rawPrNumber = process.env.PR_NUMBER;
46+
} else if (triggerSource === "workflow_dispatch") {
47+
rawPrNumber = process.env.DISPATCH_PR_NUMBER;
48+
} else {
49+
core.setFailed(`Unsupported trigger source: ${JSON.stringify(triggerSource)}`);
50+
return;
51+
}
52+
53+
const prNumber = Number(rawPrNumber);
3554
if (!Number.isInteger(prNumber) || prNumber <= 0) {
36-
core.setFailed(`workflow_dispatch input pr_number must be a positive integer, got ${JSON.stringify(process.env.PR_NUMBER)}`);
55+
core.setFailed(`${triggerSource} PR number must be a positive integer, got ${JSON.stringify(rawPrNumber)}`);
3756
return;
3857
}
3958
@@ -59,6 +78,7 @@ jobs:
5978
6079
await core.summary
6180
.addHeading("Runtime Live E2E Provenance")
81+
.addRaw(`- Trigger source: ${triggerSource}\n`)
6282
.addRaw(`- PR number: #${prNumber}\n`)
6383
.addRaw(`- Tested workflow SHA: \`${context.sha}\`\n`)
6484
.addRaw(`- Current PR head SHA: \`${pull.head.sha}\`\n`)
@@ -78,6 +98,11 @@ jobs:
7898

7999
- uses: astral-sh/setup-uv@v6
80100

101+
- name: Prepare live artifact root
102+
run: |
103+
mkdir -p "$RUNNER_TEMP/spacedock-live/$GITHUB_JOB"
104+
echo "SPACEDOCK_TEST_TMP_ROOT=$RUNNER_TEMP/spacedock-live/$GITHUB_JOB" >> "$GITHUB_ENV"
105+
81106
- name: Install Claude Code
82107
run: |
83108
curl -fsSL https://claude.ai/install.sh | bash
@@ -87,28 +112,34 @@ jobs:
87112
run: |
88113
git config --global user.name "github-actions[bot]"
89114
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
115+
git config --global init.defaultBranch main
90116
91117
- name: Show tool versions
92118
run: |
93119
claude --version
94120
uv --version
95121
96122
- name: Run Claude live suite
97-
run: |
98-
set -euo pipefail
99-
unset CLAUDECODE && uv run tests/test_gate_guardrail.py --runtime claude
100-
unset CLAUDECODE && uv run tests/test_rejection_flow.py --runtime claude
101-
unset CLAUDECODE && uv run tests/test_scaffolding_guardrail.py
102-
unset CLAUDECODE && uv run tests/test_feedback_keepalive.py
103-
unset CLAUDECODE && uv run tests/test_dispatch_completion_signal.py
104-
unset CLAUDECODE && uv run tests/test_merge_hook_guardrail.py --runtime claude
105-
unset CLAUDECODE && uv run tests/test_push_main_before_pr.py
106-
unset CLAUDECODE && uv run tests/test_rebase_branch_before_push.py
123+
run: make test-live-claude
124+
125+
- name: Upload Claude live artifacts
126+
if: always()
127+
uses: actions/upload-artifact@v4
128+
with:
129+
name: runtime-live-e2e-claude-live
130+
path: ${{ runner.temp }}/spacedock-live/${{ github.job }}
131+
if-no-files-found: warn
107132

108133
codex-live:
109134
runs-on: ubuntu-latest
135+
environment:
136+
name: CI-E2E-CODEX
137+
deployment: false
110138
env:
111-
PR_NUMBER: ${{ inputs.pr_number }}
139+
KEEP_TEST_DIR: "1"
140+
TRIGGER_SOURCE: ${{ github.event_name }}
141+
PR_NUMBER: ${{ github.event.pull_request.number }}
142+
DISPATCH_PR_NUMBER: ${{ inputs.pr_number }}
112143
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
113144
steps:
114145
- name: Check required secret
@@ -122,9 +153,21 @@ jobs:
122153
uses: actions/github-script@v7
123154
with:
124155
script: |
125-
const prNumber = Number(process.env.PR_NUMBER);
156+
const triggerSource = process.env.TRIGGER_SOURCE;
157+
let rawPrNumber;
158+
159+
if (triggerSource === "pull_request") {
160+
rawPrNumber = process.env.PR_NUMBER;
161+
} else if (triggerSource === "workflow_dispatch") {
162+
rawPrNumber = process.env.DISPATCH_PR_NUMBER;
163+
} else {
164+
core.setFailed(`Unsupported trigger source: ${JSON.stringify(triggerSource)}`);
165+
return;
166+
}
167+
168+
const prNumber = Number(rawPrNumber);
126169
if (!Number.isInteger(prNumber) || prNumber <= 0) {
127-
core.setFailed(`workflow_dispatch input pr_number must be a positive integer, got ${JSON.stringify(process.env.PR_NUMBER)}`);
170+
core.setFailed(`${triggerSource} PR number must be a positive integer, got ${JSON.stringify(rawPrNumber)}`);
128171
return;
129172
}
130173
@@ -150,6 +193,7 @@ jobs:
150193
151194
await core.summary
152195
.addHeading("Runtime Live E2E Provenance")
196+
.addRaw(`- Trigger source: ${triggerSource}\n`)
153197
.addRaw(`- PR number: #${prNumber}\n`)
154198
.addRaw(`- Tested workflow SHA: \`${context.sha}\`\n`)
155199
.addRaw(`- Current PR head SHA: \`${pull.head.sha}\`\n`)
@@ -169,22 +213,36 @@ jobs:
169213

170214
- uses: astral-sh/setup-uv@v6
171215

216+
- name: Prepare live artifact root
217+
run: |
218+
mkdir -p "$RUNNER_TEMP/spacedock-live/$GITHUB_JOB"
219+
echo "SPACEDOCK_TEST_TMP_ROOT=$RUNNER_TEMP/spacedock-live/$GITHUB_JOB" >> "$GITHUB_ENV"
220+
172221
- name: Install Codex CLI
173222
run: npm install --global @openai/codex
174223

224+
- name: Login Codex with API key
225+
run: |
226+
printenv OPENAI_API_KEY | codex login --with-api-key
227+
175228
- name: Configure git identity
176229
run: |
177230
git config --global user.name "github-actions[bot]"
178231
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
232+
git config --global init.defaultBranch main
179233
180234
- name: Show tool versions
181235
run: |
182236
codex --version
183237
uv --version
184238
185239
- name: Run Codex live suite
186-
run: |
187-
set -euo pipefail
188-
uv run tests/test_gate_guardrail.py --runtime codex
189-
uv run tests/test_rejection_flow.py --runtime codex
190-
uv run tests/test_merge_hook_guardrail.py --runtime codex
240+
run: make test-live-codex
241+
242+
- name: Upload Codex live artifacts
243+
if: always()
244+
uses: actions/upload-artifact@v4
245+
with:
246+
name: runtime-live-e2e-codex-live
247+
path: ${{ runner.temp }}/spacedock-live/${{ github.job }}
248+
if-no-files-found: warn

Makefile

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: test-static test-e2e
1+
.PHONY: test-static test-e2e test-live-claude test-live-codex
22

33
TEST ?= tests/test_gate_guardrail.py
44
RUNTIME ?= claude
@@ -8,3 +8,19 @@ test-static:
88

99
test-e2e:
1010
unset CLAUDECODE && uv run $(TEST) --runtime $(RUNTIME)
11+
12+
test-live-claude:
13+
unset CLAUDECODE && set -euo pipefail && \
14+
uv run tests/test_gate_guardrail.py --runtime claude && \
15+
uv run tests/test_rejection_flow.py --runtime claude && \
16+
uv run tests/test_feedback_keepalive.py && \
17+
uv run tests/test_merge_hook_guardrail.py --runtime claude && \
18+
uv run tests/test_push_main_before_pr.py && \
19+
uv run tests/test_rebase_branch_before_push.py
20+
# SKIPPED: test_scaffolding_guardrail.py — FO violates issue-filing guardrail. Track: file new task
21+
# SKIPPED: test_dispatch_completion_signal.py — FO drops SendMessage block. Track: #120
22+
23+
test-live-codex:
24+
uv run tests/test_gate_guardrail.py --runtime codex && \
25+
uv run tests/test_rejection_flow.py --runtime codex && \
26+
uv run tests/test_merge_hook_guardrail.py --runtime codex

docs/plans/status-workflow-discovery.md renamed to docs/plans/_archive/status-workflow-discovery.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
---
22
id: 100
33
title: "status tool: add workflow directory discovery"
4-
status: validation
4+
status: done
55
source: CL observation — Codex startup uses raw rg for discovery
66
started: 2026-04-13T15:57:56Z
7-
completed:
8-
verdict:
7+
completed: 2026-04-14T01:08:21Z
8+
verdict: PASSED
99
score: 0.60
10-
worktree: .worktrees/spacedock-ensign-status-workflow-discovery
10+
worktree:
1111
issue:
1212
pr: #85
13+
archived: 2026-04-14T01:08:37Z
1314
---
1415

1516
The startup procedure (step 2) requires searching for README.md files with `commissioned-by: spacedock@` frontmatter to discover workflow directories. Every runtime (Claude Code, Codex) reimplements this as a raw grep/rg call before it can invoke `status --boot`.

docs/plans/build-dispatch-structured-helper.md

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,40 @@ The new instructions in the `## Dispatch Adapter` section, after the team health
456456
| 4 | **Static content test invalidation.** Replacing the adapter's `Agent()` template changes the text that `test_agent_content.py` assertions match. | High (certain) | Low | The implementation must update existing assertions in `test_agent_content.py` that reference the old template wording. Specifically: `test_assembled_claude_first_officer_has_team_health_check` (AC1 asserts `test -f` which stays), `test_assembled_claude_first_officer_dispatch_template_has_team_mode_completion_signal` (must be updated to check the break-glass template or the helper instruction). Each affected test is enumerated in the implementation checklist. |
457457
| 5 | **`parse_stages_block` extension regresses existing callers.** Adding `feedback-to`, `agent`, `fresh` fields to the output dict could break callers that iterate over dict keys. | Low | Low | The extension adds optional keys to dicts in a list. Existing callers use `.get()` or explicit key access — they never iterate all keys. The implementation must verify this by grepping all `parse_stages_block` call sites. |
458458
459-
## Out of scope
459+
## Implementation Notes (gate-approved 2026-04-13)
460+
461+
The ideation gate was approved with seven follow-up items from the staff review. These are mandatory for the implementer to address; they are not optional polish.
462+
463+
1. **Correct the line references to the old dispatch template.** The ideation quotes "lines 48-57" (and in one place "lines 51-57") of `skills/first-officer/references/claude-first-officer-runtime.md` as the replacement target. The reviewer confirmed the actual `Agent(...)` block spans **lines 50-57** (line 50 opens the code block, lines 51-56 hold the Agent call, line 57 closes). Re-verify against the current file at implementation start and update any checklist or assertion that references specific line numbers.
464+
465+
2. **Spell out worktree path and branch derivation in the prompt-assembly rules.** Prompt component #3 (worktree instructions, conditional on `worktree: true`) must name where each value comes from:
466+
- `worktree_path` is read from the entity frontmatter's `worktree` field (relative or absolute — specify one and normalize).
467+
- `branch` is derived as `{worker_key}/{slug}`, where `worker_key` is the dispatch_agent_id with `:` replaced by `-` (per the existing claude-first-officer runtime adapter convention at around line 30-35).
468+
- Cross-reference to Validation Rule 4 (worktree path must exist on disk) so the reader sees which field is being validated.
469+
470+
3. **Resolve the feedback-to detection ambiguity.** Validation Rule 5 fires when the stage being dispatched is a feedback-to target, but the stdin schema as written doesn't carry an explicit "this is a feedback re-dispatch" flag. Pick one of the following and encode it in both the input schema and the rule:
471+
- **Option A (recommended):** FO sets `is_feedback_reflow: true` in stdin when routing a rejection. Helper trusts the flag. Simple, explicit.
472+
- **Option B:** Helper scans the workflow README for any stage whose `feedback-to` field matches the target stage name, then requires `feedback_context` to be present when that pattern holds. More magical; helper must read the README.
473+
- Whichever is chosen, the worked example in the Output JSON Schema section should include a feedback-reflow example alongside the ideation example.
474+
475+
4. **Add a break-glass fallback test.** AC-12 asserts the break-glass prose exists in the runtime adapter. That is not enough. Add one of:
476+
- **Unit test** in `tests/test_claude_team.py` that forces the helper to exit 1 (e.g., via malformed stdin) and asserts the FO's break-glass prose is triggered — this requires a small harness because the FO's recovery path is driven by the runtime prose, not code. If a pure unit test is not feasible, use the static-content harness to assert the break-glass template is reachable and syntactically usable as an `Agent()` call.
477+
- **Static assertion** that the break-glass template in the adapter, when rendered with stub values, produces a valid Python function call — parse it with `ast.parse` as a sanity check.
478+
479+
5. **Split the E2E test line in the test-plan table.** The "high ($2-3)" cost cell is ambiguous. Replace with two lines:
480+
- `test_structured_dispatch_happy_path_e2e` — FO assembles JSON, calls helper, forwards to Agent(), minimal worker completes. Estimated cost **$0.50–$1** on haiku/low.
481+
- `test_structured_dispatch_multistage_e2e` — deferred to #134 (runtime-specific-tests-on-pr) rather than a new standalone E2E, because the multi-stage path is already exercised by `test_dispatch_completion_signal.py` and the full FO pipeline. The implementer should confirm the existing completion-signal test goes green after the helper lands, and note this in the stage report.
482+
483+
6. **Document the FO's input-assembly guardrail.** The helper validates its input, but the FO's prose-based JSON assembly is still a failure vector: if the FO sets `bare_mode: false` when teams are not active (or vice versa), the helper rejects, the FO falls back to break-glass, and the completion signal can be lost. Either:
484+
- Add a one-sentence guardrail note in the new runtime-adapter prose at the place where the FO is told to assemble stdin: "the `bare_mode` field must match the current dispatch context — never infer it from the stage, always from the live team state." and
485+
- Add a static assertion in `tests/test_agent_content.py` that the new prose contains that guardrail sentence verbatim.
486+
487+
7. **Resolve the reuse path (SendMessage) scope question.** The runtime adapter has two dispatch surfaces: initial `Agent()` dispatch and `SendMessage(to="team-lead")` reuse-advance of an already-alive ensign. The ideation only covers the initial `Agent()` path. Decide, in this task, which of the following the implementation ships:
488+
- **Option X (narrow):** `claude-team build` serves only initial `Agent()` dispatch. Reuse continues to use the existing prose-based `SendMessage` template in the runtime adapter. The runtime adapter then has two dispatch surfaces — document this explicitly. Future task to unify.
489+
- **Option Y (wide):** `claude-team build` serves both paths. Output schema gains a `dispatch_kind: "agent"|"send_message"` field (or a sibling `claude-team send` subcommand). Single dispatch surface in the adapter.
490+
- Option X is the lower-risk landing for this task; Option Y is the cleaner end-state. Pick one before implementation starts. If Option X, add a task to the Related section filing the eventual Option Y work.
491+
492+
These notes are all implementable without another ideation cycle. Any item that cannot be resolved during implementation (e.g., discovering a structural blocker for item 7) must be raised to the captain before writing code, not silently deferred.
460493
461494
- Codex and Gemini runtime adapter updates (separate tasks after this lands).
462495
- Generalizing the pattern to other fuzzy-template sites (feedback rejection flow, gate presentation, event loop) — Phase 4 of issue #63.
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
---
2+
id: 146
3+
title: "Release-branch runtime live E2E matrix parameterization"
4+
status: backlog
5+
source: "CL direction during 2026-04-13 session — keep PR live CI on fixed defaults, but allow manual/release full-matrix runtime runs"
6+
score: 0.61
7+
started:
8+
completed:
9+
verdict:
10+
worktree:
11+
issue: #89
12+
pr:
13+
---
14+
15+
## Problem Statement
16+
17+
Task 145 moved the runtime live suite onto PRs with environment-backed approvals and split Claude/Codex environments. That gives operators a good default CI path, but it is intentionally fixed: the PR-triggered workflow runs the default jobs only, and the environment approval UI cannot collect `workflow_dispatch` inputs such as model overrides or matrix selection.
18+
19+
We still need an operator-friendly way to run broader live validation for release branches and targeted manual checks without weakening the default PR path. That follow-up should add dispatch-time parameterization for manual runs while keeping ordinary PR CI simple and stable.
20+
21+
## Recommended Approach
22+
23+
Keep the `pull_request` path opinionated and fixed. Extend `workflow_dispatch` so manual/API-triggered runs can request a broader runtime matrix for release validation.
24+
25+
The likely control surface is:
26+
27+
1. target PR/ref selection
28+
2. default vs full-matrix mode
29+
3. optional Claude model override
30+
4. optional Codex model override
31+
5. any additional release-only axes the live suite needs
32+
33+
Those values should be provided when the workflow is dispatched. Environment approval remains a separate release gate for the already-configured jobs and should not be treated as an input form.
34+
35+
## Scope Notes
36+
37+
- This task is about dispatch-time parameterization for manual and release-branch live runs.
38+
- It should not change the default PR-triggered job set for ordinary pull requests.
39+
- It should document the operator flow clearly: dispatch with inputs, approve the relevant environments, then inspect per-job artifacts and results.
40+
- It may require plumbing model inputs through the current Codex and Claude live test entrypoints so overrides actually reach the invoked runtime.
41+
42+
## Acceptance Criteria
43+
44+
1. Ordinary PR-triggered runs keep the current default runtime jobs and do not require new inputs.
45+
- Test: inspect `.github/workflows/runtime-live-e2e.yml` and confirm the `pull_request` path still runs the default jobs with no extra operator input required.
46+
2. `workflow_dispatch` supports explicit inputs for manual/release live runs, including matrix selection and optional runtime/model overrides.
47+
- Test: inspect the workflow input block and confirm the dispatched jobs derive their runtime configuration from those inputs.
48+
3. The workflow documentation explains that `workflow_dispatch` inputs are supplied at run creation time, not at environment approval time.
49+
- Test: inspect `tests/README.md` and confirm the operator caveat is documented.
50+
4. The live test entrypoints actually honor any new manual/runtime override inputs that the workflow exposes.
51+
- Test: inspect the relevant test scripts/helpers and run targeted offline checks to confirm the workflow wiring matches the harness CLI surface.
52+
5. The repo keeps offline coverage for the workflow structure and dispatch-time parameter logic.
53+
- Test: run the targeted workflow/helper tests covering the added input surface.
54+
55+
## Test Plan
56+
57+
- Workflow-structure inspection: verify the fixed `pull_request` path remains unchanged while `workflow_dispatch` gains the intended parameter surface. Cost/complexity: low. No E2E required.
58+
- Focused offline tests: update workflow/helper tests for the new input block and any runtime-argument plumbing. Cost/complexity: medium. No E2E required.
59+
- Manual release-path smoke: dispatch the workflow with a non-default matrix/model configuration and confirm the resulting jobs reflect the requested inputs before approval. Cost/complexity: medium. E2E required: yes, but only for final validation.

0 commit comments

Comments
 (0)