|
| 1 | +"""Cross-repo DOGFOOD acceptance test — the capstone of Slice 1. |
| 2 | +
|
| 3 | +This is a REAL end-to-end proof (no mocks) that the unfakable Atlas oracle gates |
| 4 | +a genuine `prd-taskmaster` ship-check. For each DONE task, the standalone |
| 5 | +`skel/ship-check.py` (Gate 5) shells the `atlas oracle grade` CLI, which: |
| 6 | +
|
| 7 | + 1. checks out the submitted commit into a throwaway detached worktree, |
| 8 | + 2. OVERLAYS the operator-held tests over the submitter's tree (the submitter's |
| 9 | + own copy of the graded path is REPLACED by the operator's held copy), |
| 10 | + 3. re-executes the card's grading command inside a digest-pinned podman sandbox, |
| 11 | + 4. derives PASS iff exit 0, and writes a tamper-evident ledger event. |
| 12 | +
|
| 13 | +The acceptance criterion is the reward-hack test: a submitter who ships a |
| 14 | +`grade.sh` that always `exit 0` (a cheat that would pass if the submitter |
| 15 | +controlled grading) does NOT ship, because the operator-held `grade.sh` |
| 16 | +(`exit 1`) is overlaid and re-executed by the oracle. The cheat never touches |
| 17 | +the verdict. |
| 18 | +
|
| 19 | +──────────────────────────────────────────────────────────────────────────────── |
| 20 | +ATLAS_ORACLE_CMD |
| 21 | + The oracle CLI lives in the SPINE worktree (a separate monorepo). Its |
| 22 | + workspace packages declare `exports: "./src/index.ts"`, so the compiled |
| 23 | + `apps/cli/dist/index.js` cannot be run by bare `node` — it would resolve the |
| 24 | + workspace deps to their TypeScript sources. We therefore invoke the CLI the |
| 25 | + same way the spine's own vitest suite does: through the `tsx` executable on |
| 26 | + the CLI source entrypoint. This requires NO edits to the spine repo and runs |
| 27 | + the identical code path. The resulting command is: |
| 28 | +
|
| 29 | + ATLAS_ORACLE_CMD="<spine>/node_modules/.bin/tsx <spine>/apps/cli/src/index.ts" |
| 30 | +
|
| 31 | + ship-check.py shlex-splits ATLAS_ORACLE_CMD and appends `oracle grade ...`. |
| 32 | +
|
| 33 | +SLICE-2 HARDENING NOTE |
| 34 | + The Graded Card's `contentHash` is a syntactically valid placeholder here. |
| 35 | + `gradeSubmission` does NOT re-verify the contentHash against the card body in |
| 36 | + Slice 1 (it is only echoed into the ledger payload), so a placeholder is |
| 37 | + accepted. Slice 2 should recompute and verify the contentHash before grading |
| 38 | + so a tampered card body cannot be graded under a stale hash. |
| 39 | +""" |
| 40 | +from __future__ import annotations |
| 41 | + |
| 42 | +import hashlib |
| 43 | +import json |
| 44 | +import os |
| 45 | +import shutil |
| 46 | +import subprocess |
| 47 | +from pathlib import Path |
| 48 | + |
| 49 | +import pytest |
| 50 | + |
| 51 | +# ── Repo paths ──────────────────────────────────────────────────────────────── |
| 52 | +ENGINE_ROOT = Path(__file__).resolve().parents[2] |
| 53 | +SKEL_SHIP_CHECK = ENGINE_ROOT / "skel" / "ship-check.py" |
| 54 | + |
| 55 | +# ── Spine (oracle CLI) paths ────────────────────────────────────────────────── |
| 56 | +SPINE_ROOT = Path("/home/anombyte/Hermes/current-projects/.worktrees/atlas-coin-oracle") |
| 57 | +TSX_BIN = SPINE_ROOT / "node_modules" / ".bin" / "tsx" |
| 58 | +CLI_SRC = SPINE_ROOT / "apps" / "cli" / "src" / "index.ts" |
| 59 | +# The oracle invocation prefix consumed by ship-check.py via ATLAS_ORACLE_CMD. |
| 60 | +CLI_CMD = f"{TSX_BIN} {CLI_SRC}" |
| 61 | + |
| 62 | +ALPINE_REF = "docker.io/library/alpine:3.20" |
| 63 | + |
| 64 | + |
| 65 | +# ── Capability probes ───────────────────────────────────────────────────────── |
| 66 | +def has_podman() -> bool: |
| 67 | + return shutil.which("podman") is not None |
| 68 | + |
| 69 | + |
| 70 | +def has_oracle_cli() -> bool: |
| 71 | + return TSX_BIN.exists() and CLI_SRC.exists() |
| 72 | + |
| 73 | + |
| 74 | +def resolve_alpine_digest() -> str: |
| 75 | + """Pull alpine:3.20 and return its `sha256:...` manifest digest.""" |
| 76 | + subprocess.run( |
| 77 | + ["podman", "pull", ALPINE_REF], |
| 78 | + check=True, capture_output=True, text=True, |
| 79 | + ) |
| 80 | + proc = subprocess.run( |
| 81 | + ["podman", "image", "inspect", ALPINE_REF, "--format", "{{.Digest}}"], |
| 82 | + check=True, capture_output=True, text=True, |
| 83 | + ) |
| 84 | + digest = proc.stdout.strip() |
| 85 | + assert digest.startswith("sha256:"), f"unexpected digest {digest!r}" |
| 86 | + return digest |
| 87 | + |
| 88 | + |
| 89 | +SKIP_REASON = "requires podman + the built atlas oracle CLI (tsx) in the spine worktree" |
| 90 | +podman_and_cli = pytest.mark.skipif( |
| 91 | + not (has_podman() and has_oracle_cli()), reason=SKIP_REASON |
| 92 | +) |
| 93 | + |
| 94 | + |
| 95 | +# ── Project fixture ─────────────────────────────────────────────────────────── |
| 96 | +def _git(args: list[str], cwd: Path) -> str: |
| 97 | + return subprocess.run( |
| 98 | + ["git", *args], cwd=cwd, check=True, capture_output=True, text=True |
| 99 | + ).stdout.strip() |
| 100 | + |
| 101 | + |
| 102 | +def make_project(tmp_path: Path, *, submitter_grade: str, operator_grade: str): |
| 103 | + """Construct a real mini-project laid out exactly as ship-check's gate_oracle |
| 104 | + expects, and return (root, head_sha). |
| 105 | +
|
| 106 | + The submitter commits their own `grade.sh` (submitter_grade). The operator's |
| 107 | + held copy (operator_grade) lives — uncommitted, on disk only — under |
| 108 | + .atlas-ai/held-out/ and is what the oracle overlays + re-executes. |
| 109 | + """ |
| 110 | + root = tmp_path |
| 111 | + digest = resolve_alpine_digest() |
| 112 | + |
| 113 | + # 1. Submitter's working tree: grade.sh committed at root. |
| 114 | + (root / "grade.sh").write_text(submitter_grade) |
| 115 | + _git(["init", "."], root) |
| 116 | + _git(["config", "user.email", "dogfood@atlas.test"], root) |
| 117 | + _git(["config", "user.name", "dogfood"], root) |
| 118 | + _git(["add", "-A"], root) |
| 119 | + # core.hooksPath=/dev/null mirrors the oracle's own git invocations and keeps |
| 120 | + # any ambient git hooks out of the committed state. |
| 121 | + _git(["-c", "core.hooksPath=/dev/null", "commit", "-m", "work"], root) |
| 122 | + head_sha = _git(["rev-parse", "HEAD"], root) |
| 123 | + |
| 124 | + atlas = root / ".atlas-ai" |
| 125 | + |
| 126 | + # 2. Operator-held test bundle (on disk; need not be committed). |
| 127 | + held = atlas / "held-out" |
| 128 | + held.mkdir(parents=True) |
| 129 | + (held / "grade.sh").write_text(operator_grade) |
| 130 | + operator_sha256 = hashlib.sha256(operator_grade.encode()).hexdigest() |
| 131 | + |
| 132 | + # 3. Graded Card. |
| 133 | + cdd = atlas / "cdd" |
| 134 | + cdd.mkdir(parents=True) |
| 135 | + card = { |
| 136 | + "id": "C-001", |
| 137 | + "taskId": 1, |
| 138 | + "title": "dogfood", |
| 139 | + "given": ["g"], |
| 140 | + "when": ["w"], |
| 141 | + "then": [{ |
| 142 | + "index": 1, |
| 143 | + "statement": "exit 0", |
| 144 | + "evidenceTier": "A", |
| 145 | + "evidenceKind": "command-output", |
| 146 | + }], |
| 147 | + "author": {"kind": "human", "id": "op"}, |
| 148 | + "createdAt": "2026-06-16T00:00:00.000Z", |
| 149 | + # SLICE-2: placeholder contentHash — gradeSubmission does not re-verify it |
| 150 | + # in Slice 1 (see module docstring). |
| 151 | + "contentHash": "sha256:" + "0" * 64, |
| 152 | + "frozenAt": "2026-06-16T00:00:00.000Z", |
| 153 | + "grading": { |
| 154 | + "command": ["sh", "grade.sh"], |
| 155 | + "heldOutTests": [{"path": "grade.sh", "sha256": operator_sha256}], |
| 156 | + "gradedPaths": ["grade.sh"], |
| 157 | + "baseImage": {"ref": ALPINE_REF, "digest": digest}, |
| 158 | + "env": { |
| 159 | + "LANG": "C", "LC_ALL": "C", "TZ": "UTC", |
| 160 | + "SOURCE_DATE_EPOCH": 0, "seed": 0, |
| 161 | + "parallelism": 1, "cpuClass": "x86-64-v2", |
| 162 | + }, |
| 163 | + "timeoutMs": 60000, |
| 164 | + }, |
| 165 | + } |
| 166 | + (cdd / "task-1.json").write_text(json.dumps(card)) |
| 167 | + |
| 168 | + # 4. Gates 1-4 scaffolding so the ORACLE gate (Gate 5) decides the verdict. |
| 169 | + state = atlas / "state" |
| 170 | + state.mkdir(parents=True) |
| 171 | + (state / "pipeline.json").write_text(json.dumps({"current_phase": "EXECUTE"})) |
| 172 | + |
| 173 | + tasks_dir = root / ".taskmaster" / "tasks" |
| 174 | + tasks_dir.mkdir(parents=True) |
| 175 | + (tasks_dir / "tasks.json").write_text( |
| 176 | + json.dumps({"master": {"tasks": [{"id": 1, "status": "done"}]}}) |
| 177 | + ) |
| 178 | + docs = root / ".taskmaster" / "docs" |
| 179 | + docs.mkdir(parents=True) |
| 180 | + (docs / "plan.md").write_text("# Plan\n") |
| 181 | + |
| 182 | + # Oracle output dirs (created by the CLI too, but explicit is clearer). |
| 183 | + (atlas / "evidence").mkdir(parents=True, exist_ok=True) |
| 184 | + (atlas / "ledger").mkdir(parents=True, exist_ok=True) |
| 185 | + |
| 186 | + return root, head_sha |
| 187 | + |
| 188 | + |
| 189 | +def run_shipcheck(root: Path) -> subprocess.CompletedProcess: |
| 190 | + """Run the REAL skel/ship-check.py as a subprocess with ATLAS_ORACLE_CMD set.""" |
| 191 | + return subprocess.run( |
| 192 | + ["python3", str(SKEL_SHIP_CHECK), "--cwd", str(root)], |
| 193 | + capture_output=True, |
| 194 | + text=True, |
| 195 | + env={**os.environ, "ATLAS_ORACLE_CMD": CLI_CMD}, |
| 196 | + ) |
| 197 | + |
| 198 | + |
| 199 | +def _read_ledger_event(root: Path) -> dict: |
| 200 | + ledger = root / ".atlas-ai" / "ledger" |
| 201 | + events = sorted(ledger.glob("*.json")) |
| 202 | + assert events, f"no ledger event written under {ledger}" |
| 203 | + return json.loads(events[-1].read_text()) |
| 204 | + |
| 205 | + |
| 206 | +# ── Test 1: genuine pass ships ──────────────────────────────────────────────── |
| 207 | +@podman_and_cli |
| 208 | +def test_genuine_pass_ships(tmp_path): |
| 209 | + # submitter_grade is irrelevant — the operator-held grade.sh is overlaid. |
| 210 | + root, _head = make_project(tmp_path, submitter_grade="exit 1\n", operator_grade="exit 0\n") |
| 211 | + |
| 212 | + proc = run_shipcheck(root) |
| 213 | + |
| 214 | + assert "SHIP_CHECK_OK" in proc.stdout, ( |
| 215 | + f"expected SHIP_CHECK_OK; rc={proc.returncode}\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" |
| 216 | + ) |
| 217 | + assert proc.returncode == 0, f"rc={proc.returncode}\nSTDERR:\n{proc.stderr}" |
| 218 | + |
| 219 | + # The oracle recorded a PASS verdict in the tamper-evident ledger. |
| 220 | + event = _read_ledger_event(root) |
| 221 | + assert event["type"] == "verification.completed" |
| 222 | + assert event["payload"]["verdict"] == "PASS", event["payload"] |
| 223 | + |
| 224 | + |
| 225 | +# ── Test 2: reward hack blocked (THE acceptance criterion) ──────────────────── |
| 226 | +@podman_and_cli |
| 227 | +def test_reward_hack_blocked(tmp_path): |
| 228 | + # The submitter ships a cheat: grade.sh that always passes. The operator-held |
| 229 | + # grade.sh fails. Because the oracle overlays + re-executes the operator copy, |
| 230 | + # the cheat does NOT ship. |
| 231 | + root, _head = make_project(tmp_path, submitter_grade="exit 0\n", operator_grade="exit 1\n") |
| 232 | + |
| 233 | + proc = run_shipcheck(root) |
| 234 | + |
| 235 | + assert "SHIP_CHECK_OK" not in proc.stdout, ( |
| 236 | + f"REWARD HACK SHIPPED — cheat was not blocked!\nSTDOUT:\n{proc.stdout}" |
| 237 | + ) |
| 238 | + assert proc.returncode != 0, "cheat must produce a non-zero exit" |
| 239 | + # The blocked-FAIL is surfaced on stderr for task 1. |
| 240 | + assert "oracle" in proc.stderr.lower() and "1" in proc.stderr, ( |
| 241 | + f"expected an oracle FAIL for task 1 on stderr\nSTDERR:\n{proc.stderr}" |
| 242 | + ) |
| 243 | + |
| 244 | + # And the ledger records the FAIL verdict — the operator-held test was run. |
| 245 | + event = _read_ledger_event(root) |
| 246 | + assert event["payload"]["verdict"] == "FAIL", event["payload"] |
| 247 | + |
| 248 | + |
| 249 | +# ── Test 3: ledger integrity ────────────────────────────────────────────────── |
| 250 | +@podman_and_cli |
| 251 | +def test_ledger_integrity(tmp_path): |
| 252 | + """After a genuine PASS, the tamper-evident ledger verifies clean. |
| 253 | +
|
| 254 | + The spine CLI exposes `ledger verify <dir>` (positional arg). If that |
| 255 | + subcommand is unavailable we fall back to re-parsing the event file. |
| 256 | + """ |
| 257 | + root, _head = make_project(tmp_path, submitter_grade="exit 1\n", operator_grade="exit 0\n") |
| 258 | + proc = run_shipcheck(root) |
| 259 | + assert "SHIP_CHECK_OK" in proc.stdout, proc.stderr |
| 260 | + |
| 261 | + ledger_dir = root / ".atlas-ai" / "ledger" |
| 262 | + verify = subprocess.run( |
| 263 | + [str(TSX_BIN), str(CLI_SRC), "ledger", "verify", str(ledger_dir)], |
| 264 | + capture_output=True, text=True, |
| 265 | + ) |
| 266 | + if verify.returncode == 0 and verify.stdout.strip(): |
| 267 | + result = json.loads(verify.stdout) |
| 268 | + assert result.get("ok") is True, f"ledger verify reported {result!r}" |
| 269 | + assert result.get("eventCount", 0) >= 1, result |
| 270 | + else: |
| 271 | + # No usable `ledger verify` subcommand — don't fail the task over it; |
| 272 | + # confirm the event file at least parses and records the PASS. |
| 273 | + event = _read_ledger_event(root) |
| 274 | + assert event["payload"]["verdict"] == "PASS", event["payload"] |
0 commit comments