ci: preserve runtime live test artifacts

Author: clkao

.github/workflows/runtime-live-e2e.yml

@@ -19,6 +19,8 @@ jobs:
     environment: CI-E2E
     env:
       DISABLE_AUTOUPDATER: "1"
+      KEEP_TEST_DIR: "1"
+      SPACEDOCK_TEST_TMP_ROOT: ${{ runner.temp }}/spacedock-live/${{ github.job }}
       TRIGGER_SOURCE: ${{ github.event_name }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
       DISPATCH_PR_NUMBER: ${{ inputs.pr_number }}
@@ -122,10 +124,20 @@ jobs:
           unset CLAUDECODE && uv run tests/test_push_main_before_pr.py
           unset CLAUDECODE && uv run tests/test_rebase_branch_before_push.py
 
+      - name: Upload Claude live artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtime-live-e2e-claude-live
+          path: ${{ env.SPACEDOCK_TEST_TMP_ROOT }}
+          if-no-files-found: warn
+
   codex-live:
     runs-on: ubuntu-latest
     environment: CI-E2E
     env:
+      KEEP_TEST_DIR: "1"
+      SPACEDOCK_TEST_TMP_ROOT: ${{ runner.temp }}/spacedock-live/${{ github.job }}
       TRIGGER_SOURCE: ${{ github.event_name }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
       DISPATCH_PR_NUMBER: ${{ inputs.pr_number }}
@@ -221,3 +233,11 @@ jobs:
           uv run tests/test_gate_guardrail.py --runtime codex
           uv run tests/test_rejection_flow.py --runtime codex
           uv run tests/test_merge_hook_guardrail.py --runtime codex
+
+      - name: Upload Codex live artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtime-live-e2e-codex-live
+          path: ${{ env.SPACEDOCK_TEST_TMP_ROOT }}
+          if-no-files-found: warn

docs/superpowers/plans/2026-04-13-runtime-live-e2e-artifact-preservation.md

@@ -0,0 +1,97 @@
+# Runtime Live E2E Artifact Preservation Implementation Plan
+
+> **For Claude:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Preserve and upload live-test temp directories as GitHub Actions artifacts on every runtime live E2E run.
+
+**Architecture:** Add a deterministic temp-root hook to the shared live-test harness, then point both live workflow jobs at job-specific temp roots and upload those directories with `actions/upload-artifact`. Keep default local behavior unchanged when the CI env hook is absent.
+
+**Tech Stack:** Python test harness, GitHub Actions workflow YAML, pytest-style offline workflow checks
+
+---
+
+### Task 1: Add a failing helper test for deterministic temp roots
+
+**Files:**
+- Modify: `tests/test_test_lib_helpers.py`
+- Modify: `scripts/test_lib.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Add a unit test asserting `TestRunner` creates `test_dir` under `SPACEDOCK_TEST_TMP_ROOT` when that env var is set.
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `uv run --with pytest python -m pytest tests/test_test_lib_helpers.py -q`
+Expected: FAIL because `TestRunner` ignores `SPACEDOCK_TEST_TMP_ROOT`.
+
+- [ ] **Step 3: Write minimal implementation**
+
+Update `TestRunner` to use `tempfile.mkdtemp(dir=..., prefix=...)` when the env var is present.
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `uv run --with pytest python -m pytest tests/test_test_lib_helpers.py -q`
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add tests/test_test_lib_helpers.py scripts/test_lib.py
+git commit -m "test: preserve live test dirs under a configured root"
+```
+
+### Task 2: Add a failing workflow test for artifact preservation
+
+**Files:**
+- Modify: `tests/test_runtime_live_e2e_workflow.py`
+- Modify: `.github/workflows/runtime-live-e2e.yml`
+
+- [ ] **Step 1: Write the failing test**
+
+Add assertions that each live job sets `KEEP_TEST_DIR`, sets `SPACEDOCK_TEST_TMP_ROOT`, and uploads artifacts on `if: always()`.
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `unset CLAUDECODE && uv run tests/test_runtime_live_e2e_workflow.py`
+Expected: FAIL because the workflow does not yet preserve or upload live temp dirs.
+
+- [ ] **Step 3: Write minimal implementation**
+
+Set the env vars in both live jobs and add upload-artifact steps after the live suite commands.
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `unset CLAUDECODE && uv run tests/test_runtime_live_e2e_workflow.py`
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add tests/test_runtime_live_e2e_workflow.py .github/workflows/runtime-live-e2e.yml
+git commit -m "ci: upload preserved live test dirs"
+```
+
+### Task 3: Run focused verification
+
+**Files:**
+- Modify: `tests/README.md`
+
+- [ ] **Step 1: Update docs**
+
+Document that live workflow runs preserve test dirs as artifacts.
+
+- [ ] **Step 2: Run focused verification**
+
+Run:
+- `uv run --with pytest python -m pytest tests/test_test_lib_helpers.py -q`
+- `unset CLAUDECODE && uv run tests/test_runtime_live_e2e_workflow.py`
+
+Expected: both pass.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add tests/README.md
+git commit -m "docs: document live test artifact preservation"
+```

docs/superpowers/specs/2026-04-13-runtime-live-e2e-artifact-preservation-design.md

@@ -0,0 +1,29 @@
+# Runtime Live E2E Artifact Preservation Design
+
+**Goal:** Preserve each live test's temp directory in CI and upload it as a GitHub Actions artifact on every `Runtime Live E2E` run.
+
+**Why:** The current live workflow only exposes the outer job log. When a live test fails in CI, the per-test temp directory printed by `TestRunner` is lost with the runner filesystem, which blocks diagnosis of CI-only failures.
+
+## Scope
+
+- Preserve temp directories for both `claude-live` and `codex-live`.
+- Store preserved test dirs under a deterministic per-job root instead of anonymous `mkdtemp()` paths.
+- Upload those preserved directories as workflow artifacts on every run, not only on failure.
+
+## Design
+
+1. `scripts/test_lib.py` will honor a new environment variable, `SPACEDOCK_TEST_TMP_ROOT`.
+   - When set, `TestRunner` will create its temp dir under that root with a stable prefix.
+   - Existing behavior remains unchanged when the variable is absent.
+
+2. `.github/workflows/runtime-live-e2e.yml` will set:
+   - `KEEP_TEST_DIR=1`
+   - `SPACEDOCK_TEST_TMP_ROOT` to a per-job path under `${{ runner.temp }}/spacedock-live/${{ github.job }}`
+
+3. The workflow will upload `${{ env.SPACEDOCK_TEST_TMP_ROOT }}` with `actions/upload-artifact` using `if: always()`.
+   - Artifact names should distinguish `claude-live` and `codex-live`.
+
+## Validation
+
+- Offline workflow test should assert `KEEP_TEST_DIR`, `SPACEDOCK_TEST_TMP_ROOT`, and upload-artifact steps are present for both live jobs.
+- Helper unit test should assert `TestRunner` creates temp dirs under the configured root.

scripts/test_lib.py

@@ -385,7 +385,15 @@ def __init__(self, test_name: str, keep_test_dir: bool = False):
         self.passes = 0
         self.failures = 0
         self.repo_root = Path(__file__).resolve().parent.parent
-        self.test_dir = Path(tempfile.mkdtemp())
+        temp_root = os.environ.get("SPACEDOCK_TEST_TMP_ROOT")
+        if temp_root:
+            temp_root_path = Path(temp_root)
+            temp_root_path.mkdir(parents=True, exist_ok=True)
+            self.test_dir = Path(
+                tempfile.mkdtemp(prefix="spacedock-test-", dir=str(temp_root_path))
+            )
+        else:
+            self.test_dir = Path(tempfile.mkdtemp())
         self.log_dir = self.test_dir
         self.keep_test_dir = keep_test_dir or bool(os.environ.get("KEEP_TEST_DIR"))
         self.test_project_dir: Path | None = None

tests/README.md

@@ -226,7 +226,12 @@ Operators should expect each job summary to show the run provenance explicitly:
 - same-repo vs fork status
 - approval/reviewer context
 
-Set `KEEP_TEST_DIR=1` to preserve temp directories after test runs for debugging.
+The live workflow sets `KEEP_TEST_DIR=1` automatically and uploads each job's preserved temp dirs as GitHub Actions artifacts:
+
+- `runtime-live-e2e-claude-live`
+- `runtime-live-e2e-codex-live`
+
+For local debugging, set `KEEP_TEST_DIR=1` to preserve temp directories after test runs. Set `SPACEDOCK_TEST_TMP_ROOT=/path/to/root` to force `TestRunner` to create preserved dirs under a predictable parent directory.
 
 ## File Requirements
 

tests/test_runtime_live_e2e_workflow.py

@@ -57,6 +57,24 @@ def test_runtime_live_e2e_workflow_has_exactly_two_runtime_jobs():
     assert "matrix:" not in text
 
 
+def test_runtime_live_e2e_workflow_preserves_and_uploads_live_test_dirs():
+    text = read_workflow()
+    claude_section = section(text, "  claude-live")
+    codex_section = section(text, "  codex-live")
+
+    for job_section, artifact_name in (
+        (claude_section, "runtime-live-e2e-claude-live"),
+        (codex_section, "runtime-live-e2e-codex-live"),
+    ):
+        assert 'KEEP_TEST_DIR: "1"' in job_section
+        assert "SPACEDOCK_TEST_TMP_ROOT:" in job_section
+        assert "${{ runner.temp }}/spacedock-live/${{ github.job }}" in job_section
+        assert "uses: actions/upload-artifact@v4" in job_section
+        assert "if: always()" in job_section
+        assert f"name: {artifact_name}" in job_section
+        assert "path: ${{ env.SPACEDOCK_TEST_TMP_ROOT }}" in job_section
+
+
 def test_runtime_live_e2e_workflow_scopes_secrets_to_the_matching_job():
     text = read_workflow()
     claude_section = section(text, "  claude-live")

tests/test_test_lib_helpers.py

@@ -14,7 +14,7 @@
 import pytest
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
-from test_lib import bash_command_targets_write, emit_skip_result, probe_claude_runtime
+from test_lib import TestRunner, bash_command_targets_write, emit_skip_result, probe_claude_runtime
 
 
 TARGETS = ("skills/", "agents/", "references/", "plugin.json")
@@ -112,3 +112,13 @@ def test_emit_skip_result_prints_standardized_skip_output(capsys):
     assert "SKIP: runtime unavailable" in captured
     assert "RESULT: SKIP" in captured
     assert excinfo.value.code == 0
+
+
+def test_test_runner_uses_configured_temp_root(monkeypatch, tmp_path):
+    configured_root = tmp_path / "live-artifacts"
+    monkeypatch.setenv("SPACEDOCK_TEST_TMP_ROOT", str(configured_root))
+
+    runner = TestRunner("helper temp root", keep_test_dir=True)
+
+    assert runner.test_dir.parent == configured_root
+    assert runner.test_dir.name.startswith("spacedock-test-")