Skip to content

Commit a88d4f6

Browse files
clkaoSpike Test
clkao
and
Spike Test
authored
e2e-gate --status success filter hides re-run-to-green Runtime Live E2E runs from the release gate (#460)
* e2e-gate: drop --status success from ghRunListForCommit At the v0.23.0 cut, gh run list --status success returned [] for a run re-run to green minutes earlier (conclusion success, attempt 2), while the same query without --status already saw the success -- the filter lags conclusion right after a re-run, blocking a genuinely green cut. The parked-run defense the flag was meant to provide already lives in EvaluateE2EGate's predicate (empty conclusion never matches), so the flag was redundant and only cost re-run-to-green cuts. T1 drives the real ghRunListForCommit against a fake gh shim pinning the gate-time observation (RED under --status, GREEN once dropped). T2 confirms the parked-run defense survives the flag's removal. * e2e-gate test: pin --workflow "Runtime Live E2E" as the query's live-matrix binding Cycle-1 audit finding: workflowName is not among ghRunListForCommit's fetched --json fields, so the --workflow "Runtime Live E2E" argv token is the ONLY thing binding the release gate to the live matrix. Dropping it left the whole suite green -- any green run on the release commit (e.g. ordinary push CI) would then satisfy the gate. The gh shim now returns [] unless argv carries that exact --workflow "Runtime Live E2E" pair, so the token's removal fails the gate closed in the test. Verified locally: removing the token from ghRunListForCommit flips TestE2EGateCommandPassesOnRerunToGreenLiveRun RED (exit 1, "no conclusion:success..."); reverting restores GREEN. --------- Co-authored-by: Spike Test <spike@example.com>
1 parent 972e2fb commit a88d4f6

4 files changed

Lines changed: 118 additions & 7 deletions

File tree

‎cmd/spacedock-release/e2e_gate.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,19 @@ import (
1515
// without a live gh.
1616
type runListFunc func(commit string) ([]byte, error)
1717

18-
// ghRunListForCommit queries the green Runtime Live E2E runs whose head commit is
19-
// the release commit. `--status success` excludes parked/waiting runs (the spike
20-
// proved a parked run is never success), and `-c <commit>` binds the query to the
21-
// exact tagged commit so a green run on some other line cannot satisfy the gate.
18+
// ghRunListForCommit queries the Runtime Live E2E runs whose head commit is the
19+
// release commit; `-c <commit>` binds the query to the exact tagged commit so a
20+
// green run on some other line cannot satisfy the gate. The query deliberately
21+
// omits `--status success`: gh's `--status` filter can lag a run's `conclusion`
22+
// right after a re-run flips it to green (observed at the v0.23.0 cut — the
23+
// filtered query returned `[]` while the same query without `--status` already
24+
// saw the success), so the pre-filter costs genuine re-run-to-green cuts and
25+
// buys nothing. Excluding parked/waiting runs is EvaluateE2EGate's job: its
26+
// predicate requires conclusion == "success", so an empty-conclusion parked run
27+
// never qualifies regardless of what this query returns.
2228
func ghRunListForCommit(commit string) ([]byte, error) {
2329
cmd := exec.Command("gh", "run", "list",
2430
"--workflow", "Runtime Live E2E",
25-
"--status", "success",
2631
"-c", commit,
2732
"--json", "databaseId,headSha,conclusion,status",
2833
)

‎cmd/spacedock-release/e2e_gate_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"os"
55
"path/filepath"
6+
"runtime"
67
"strings"
78
"testing"
89
)
@@ -82,6 +83,109 @@ func TestE2EGateCommandRejectsMissingCommit(t *testing.T) {
8283
}
8384
}
8485

86+
// writeGhStatusLagShim writes a `gh` shim that reproduces the v0.23.0 gate-time
87+
// filter-consistency lag: when argv carries a bare `--status` flag it prints
88+
// `[]` (the observed behavior of `--status success` right after a re-run flips
89+
// a run to green), and otherwise it prints fixtureJSON (the unfiltered query,
90+
// which already saw the success). This drives the REAL ghRunListForCommit
91+
// end-to-end — the fakeRunLister seam above bypasses the args entirely and
92+
// cannot see the bug.
93+
//
94+
// It also pins the query's ONLY binding to the live matrix: `--workflow
95+
// "Runtime Live E2E"` is not among the fetched `--json` fields, so nothing else
96+
// in the predicate distinguishes a Runtime Live E2E run from an unrelated green
97+
// run on the release commit (e.g. ordinary push CI). The shim prints `[]`
98+
// unless argv carries that exact `--workflow "Runtime Live E2E"` pair, so
99+
// dropping the token from ghRunListForCommit fails the gate closed here — a
100+
// regression that dropped it would otherwise fail OPEN (any green run on the
101+
// commit would satisfy the gate).
102+
func writeGhStatusLagShim(t *testing.T, fixtureJSON string) string {
103+
t.Helper()
104+
if runtime.GOOS == "windows" {
105+
t.Skip("gh stub shim is a POSIX shell script")
106+
}
107+
dir := t.TempDir()
108+
// echo, not cat: the stub PATH contains only this shim, so an external `cat`
109+
// would itself fail to resolve. echo is a shell builtin.
110+
script := `#!/bin/sh
111+
have_workflow=0
112+
prev=""
113+
for arg in "$@"; do
114+
if [ "$prev" = "--workflow" ] && [ "$arg" = "Runtime Live E2E" ]; then
115+
have_workflow=1
116+
fi
117+
if [ "$arg" = "--status" ]; then
118+
echo '[]'
119+
exit 0
120+
fi
121+
prev="$arg"
122+
done
123+
if [ "$have_workflow" != "1" ]; then
124+
echo '[]'
125+
exit 0
126+
fi
127+
echo '` + fixtureJSON + `'
128+
`
129+
path := filepath.Join(dir, "gh")
130+
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
131+
t.Fatal(err)
132+
}
133+
return dir
134+
}
135+
136+
// rerunToGreenJSON pins the 2026-06-30 gate-time observation: the only success
137+
// entry for the release commit is a run that was re-run to green (attempt 2).
138+
const rerunToGreenJSON = `[
139+
{"databaseId": 28429490220, "headSha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", "conclusion": "success", "status": "completed", "attempt": 2}
140+
]`
141+
142+
// parkedOnlyJSON has no success entry at all — only a parked/waiting run.
143+
const parkedOnlyJSON = `[
144+
{"databaseId": 28400000000, "headSha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", "conclusion": "", "status": "waiting"}
145+
]`
146+
147+
// TestE2EGateCommandPassesOnRerunToGreenLiveRun is T1 (AC-1): drives the REAL
148+
// ghRunListForCommit against the gh shim. RED under today's query shape (argv
149+
// carries `--status success`, so the shim replays the filter-consistency lag
150+
// and returns `[]`, blocking a genuinely green cut); GREEN once
151+
// ghRunListForCommit drops `--status success` (the shim then sees no bare
152+
// `--status` arg and returns the fixture, so the gate matches the run).
153+
func TestE2EGateCommandPassesOnRerunToGreenLiveRun(t *testing.T) {
154+
stubDir := writeGhStatusLagShim(t, rerunToGreenJSON)
155+
t.Setenv("PATH", stubDir)
156+
157+
summary := newSummaryFile(t)
158+
t.Setenv("GITHUB_STEP_SUMMARY", summary)
159+
t.Setenv("SPACEDOCK_E2E_GATE_WAIVER", "")
160+
161+
code := runE2EGate([]string{gateCommit}, ghRunListForCommit)
162+
if code != 0 {
163+
t.Fatalf("e2e-gate exit = %d, want 0 for a re-run-to-green Runtime Live E2E run (AC-1)", code)
164+
}
165+
if got := readFile(t, summary); !strings.Contains(got, "28429490220") {
166+
t.Errorf("step summary did not cite the matched re-run-to-green run:\n%s", got)
167+
}
168+
}
169+
170+
// TestE2EGateCommandBlocksParkedOnlyRunViaLiveQuery is T2 (AC-2): a
171+
// parked-only fixture still blocks the cut under the new (unfiltered) query
172+
// shape. The parked-run defense lives in EvaluateE2EGate's predicate (empty
173+
// conclusion never matches), not in the dropped `--status` flag, so removing
174+
// the flag must not regress this.
175+
func TestE2EGateCommandBlocksParkedOnlyRunViaLiveQuery(t *testing.T) {
176+
stubDir := writeGhStatusLagShim(t, parkedOnlyJSON)
177+
t.Setenv("PATH", stubDir)
178+
179+
summary := newSummaryFile(t)
180+
t.Setenv("GITHUB_STEP_SUMMARY", summary)
181+
t.Setenv("SPACEDOCK_E2E_GATE_WAIVER", "")
182+
183+
code := runE2EGate([]string{gateCommit}, ghRunListForCommit)
184+
if code == 0 {
185+
t.Fatalf("e2e-gate exit = 0 on a parked-only run list; want non-zero (cut blocked, AC-2)")
186+
}
187+
}
188+
85189
func newSummaryFile(t *testing.T) string {
86190
t.Helper()
87191
path := filepath.Join(t.TempDir(), "step_summary")

‎docs/releasing.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ green Runtime Live E2E run for its exact SHA. Stamp and push the release commit
9393
4. Green that exact commit. Capture the release SHA, then dispatch Runtime Live
9494
E2E on it and wait for a `conclusion: success` run — the `e2e-gate` matches a
9595
green run to the tagged SHA, and the workflow is `workflow_dispatch`-only, so
96-
nothing greens the commit unless you dispatch it:
96+
nothing greens the commit unless you dispatch it. A lane that flakes can be
97+
re-run to green (`gh run rerun <run-id> --failed`); the re-run-to-green run
98+
satisfies the gate — no fresh dispatch needed:
9799

98100
```bash
99101
REL_SHA=$(git rev-parse HEAD)

‎internal/release/e2egate_workflow_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
// - the goreleaser-carrying job declares `needs:` including `e2e-gate`, and
1414
// - the e2e-gate job resolves the tagged commit SHA and runs the
1515
// `spacedock-release e2e-gate` step over that SHA (which itself queries
16-
// `gh run list --workflow "Runtime Live E2E" --status success -c <sha>`).
16+
// `gh run list --workflow "Runtime Live E2E" -c <sha>`).
1717
func TestReleaseWorkflowGatesGoreleaserOnE2E(t *testing.T) {
1818
release := readWorkflow(t, "release.yml")
1919
if err := assertReleaseWorkflowGatesGoreleaserOnE2E(release); err != nil {

0 commit comments

Comments
 (0)