From 2a0adc09be7dff34c7d894c1664c81df63000845 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:01:24 -0700 Subject: [PATCH 1/7] feat(doctor): add _check_codex_graduation as Check 33 Add a new doctor check that evaluates four graduation criteria for the codex_backend feature: version check, probe-harness cache, matrix last-run, and sessions.jsonl smoke run. Returns Severity.INFO with an EXPERIMENTAL hold note when any criterion is non-green. Skips with Severity.OK when no codex backend is configured. - Add _check_codex_graduation to _doctor_runtime.py - Register as Check 33 in __init__.py Co-Authored-By: Claude Fable 5 --- src/autoskillit/cli/doctor/__init__.py | 4 ++ src/autoskillit/cli/doctor/_doctor_runtime.py | 70 ++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/autoskillit/cli/doctor/__init__.py b/src/autoskillit/cli/doctor/__init__.py index eeca2a1250..fd293adf2d 100644 --- a/src/autoskillit/cli/doctor/__init__.py +++ b/src/autoskillit/cli/doctor/__init__.py @@ -61,6 +61,7 @@ ) from ._doctor_runtime import ( _check_claude_process_state_breakdown, + _check_codex_graduation, _check_codex_version, _check_quota_cache_schema, _check_script_binary, @@ -213,6 +214,9 @@ def run_doctor(*, output_json: bool = False) -> None: _check_codex_mcp_timeouts(backend=_backend, run_skill=cfg.run_skill, fleet=cfg.fleet) ) + # Check 33: Codex graduation criteria + results.append(_check_codex_graduation(backend=_backend)) + # Output if output_json: print( diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index f31dbd544c..1098dedf43 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -8,7 +8,7 @@ import regex as re -from autoskillit.core import CodingAgentBackend, Severity, get_logger +from autoskillit.core import CodingAgentBackend, Severity, default_log_dir, get_logger from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION from ._doctor_types import DoctorResult @@ -192,3 +192,71 @@ def _check_script_binary() -> DoctorResult: "script(1) present but -qefc flags unsupported — PTY wrapping may not work correctly", ) return DoctorResult(Severity.OK, check_name, "script(1) available with -qefc support") + + +def _check_codex_graduation( + *, + backend: CodingAgentBackend | None = None, + log_dir: Path | None = None, +) -> DoctorResult: + check_name = "codex_graduation" + + if backend is None or not backend.capabilities.version_check_command: + return DoctorResult(Severity.OK, check_name, "Skipped (no codex backend)") + + log_root = log_dir or default_log_dir() + + # Criterion 1: version check + version_result = _check_codex_version(backend=backend) + version_status = "pass" if version_result.severity == Severity.OK else "fail" + + # Criterion 2: probe-harness cache + probe_path = log_root / "codex-probe-cache.json" + probe_status = "not-yet-run" + try: + raw = json.loads(probe_path.read_text()) + entries = raw.get("entries", {}) if isinstance(raw, dict) else {} + if entries: + probe_status = "pass" if any(e.get("passed") for e in entries.values()) else "fail" + except (OSError, json.JSONDecodeError, ValueError): + pass + + # Criterion 3: matrix last-run result + matrix_path = log_root / "codex-matrix-result.json" + matrix_status = "not-yet-run" + try: + matrix_raw = json.loads(matrix_path.read_text()) + if isinstance(matrix_raw, dict) and "passed" in matrix_raw: + matrix_status = "pass" if matrix_raw["passed"] else "fail" + except (OSError, json.JSONDecodeError, ValueError): + pass + + # Criterion 4: sessions.jsonl smoke + sessions_path = log_root / "sessions.jsonl" + smoke_status = "not-found" + try: + for line in reversed(sessions_path.read_text().splitlines()): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if entry.get("backend") == "codex": + smoke_status = "pass" if entry.get("success") else "fail" + break + except OSError: + pass + + statuses = [version_status, probe_status, matrix_status, smoke_status] + summary = ( + f"version={version_status} | probe={probe_status}" + f" | matrix={matrix_status} | smoke={smoke_status}" + ) + + if not all(s == "pass" for s in statuses): + pending = sum(1 for s in statuses if s != "pass") + summary += f" — EXPERIMENTAL hold: {pending} of 4 criteria pending" + + return DoctorResult(Severity.INFO, check_name, summary) From 96888b524d564a8959fd7ea2e6fb82c4fbd0a250 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:01:28 -0700 Subject: [PATCH 2/7] test(doctor): add TestCheckCodexGraduation covering Check 33 Add five test scenarios for _check_codex_graduation: - test_skip_for_none_backend - test_skip_for_non_codex_backend - test_all_green_omits_hold_note - test_non_green_includes_hold_note - test_absent_files_yield_not_yet_run All tests use tmp_path for filesystem isolation and monkeypatch for subprocess stubbing. Add module-level json import. Co-Authored-By: Claude Fable 5 --- tests/cli/test_doctor_backend_guards.py | 112 ++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/tests/cli/test_doctor_backend_guards.py b/tests/cli/test_doctor_backend_guards.py index 4df90dd59d..32bfa00920 100644 --- a/tests/cli/test_doctor_backend_guards.py +++ b/tests/cli/test_doctor_backend_guards.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace @@ -579,3 +580,114 @@ def test_version_above_minimum_returns_ok(self, monkeypatch: pytest.MonkeyPatch) result = _check_codex_version(backend=CodexBackend()) assert result.severity == Severity.OK assert "0.131.0" in result.message + + +class TestCheckCodexGraduation: + """Tests for _check_codex_graduation doctor check (Check 33).""" + + def _codex_result(self, stdout: str, returncode: int = 0, stderr: str = ""): + return type( + "CompletedProcess", + (), + {"returncode": returncode, "stdout": stdout, "stderr": stderr}, + )() + + def test_skip_for_none_backend(self) -> None: + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + + result = _check_codex_graduation(backend=None) + assert result.severity == Severity.OK + assert result.check == "codex_graduation" + assert "skipped" in result.message.lower() + + def test_skip_for_non_codex_backend(self) -> None: + from types import SimpleNamespace + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + + stub = SimpleNamespace(capabilities=SimpleNamespace(version_check_command="")) + result = _check_codex_graduation(backend=stub) + assert result.severity == Severity.OK + assert "skipped" in result.message.lower() + + def test_all_green_omits_hold_note( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + from datetime import UTC, datetime + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + ts = datetime.now(UTC).isoformat() + (tmp_path / "codex-probe-cache.json").write_text( + json.dumps( + { + "schema_version": 1, + "entries": {"0.130.0": {"passed": True, "probe_timestamp": ts}}, + } + ) + ) + (tmp_path / "codex-matrix-result.json").write_text(json.dumps({"passed": True})) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"backend": "codex", "success": True}) + "\n" + ) + + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "version=pass" in result.message + assert "probe=pass" in result.message + assert "matrix=pass" in result.message + assert "smoke=pass" in result.message + assert "EXPERIMENTAL" not in result.message + + def test_non_green_includes_hold_note( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"backend": "codex", "success": True}) + "\n" + ) + + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "not-yet-run" in result.message + assert "EXPERIMENTAL" in result.message + + def test_absent_files_yield_not_yet_run( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "probe=not-yet-run" in result.message + assert "matrix=not-yet-run" in result.message + assert "smoke=not-found" in result.message From 85de57c86c1c0394a46113ea5b2488a63fb1d5a5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:01:31 -0700 Subject: [PATCH 3/7] docs(doctor): update AGENTS.md to reflect new codex graduation check Bump doctor check count from 28 to 33 in src/autoskillit/cli/doctor/AGENTS.md and add codex graduation to the _doctor_runtime.py description. Update tests/cli/AGENTS.md to mention codex graduation in test_doctor_backend_guards.py description. Co-Authored-By: Claude Fable 5 --- src/autoskillit/cli/doctor/AGENTS.md | 4 ++-- tests/cli/AGENTS.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/cli/doctor/AGENTS.md b/src/autoskillit/cli/doctor/AGENTS.md index 1fe02444db..1598899c06 100644 --- a/src/autoskillit/cli/doctor/AGENTS.md +++ b/src/autoskillit/cli/doctor/AGENTS.md @@ -1,6 +1,6 @@ # doctor/ -Diagnostic health checks for the autoskillit installation (28 checks). +Diagnostic health checks for the autoskillit installation (33 checks). ## Files @@ -15,7 +15,7 @@ Diagnostic health checks for the autoskillit installation (28 checks). | `_doctor_hooks.py` | Hook registration, executability, and registry drift checks | | `_doctor_install.py` | Install path, entry points, version drift, update dismissal checks | | `_doctor_mcp.py` | MCP server registration, dual registration, plugin cache checks | -| `_doctor_runtime.py` | Quota cache schema version, claude process state, and codex version checks | +| `_doctor_runtime.py` | Quota cache schema version, claude process state, codex version, and codex graduation checks | ## Architecture Notes diff --git a/tests/cli/AGENTS.md b/tests/cli/AGENTS.md index d0d8298361..23172ab849 100644 --- a/tests/cli/AGENTS.md +++ b/tests/cli/AGENTS.md @@ -28,7 +28,7 @@ CLI command, subcommand, and interactive workflow tests. | `test_cook_order_prompt.py` | Tests: cook CLI order command — system prompt content, MCP prefix selection, ownership | | `test_cook_workspace.py` | Tests: cook CLI workspace init and clean commands | | `test_doctor.py` | Tests for CLI doctor command and related utilities | -| `test_doctor_backend_guards.py` | Tests for doctor backend guard checks (stale MCP, MCP registered, process state) and run_doctor backend wiring | +| `test_doctor_backend_guards.py` | Tests for doctor backend guard checks (stale MCP, MCP registered, process state, codex graduation) and run_doctor backend wiring | | `test_doctor_codex_mcp.py` | Tests for _check_mcp_server_registered Codex branch — monkeypatch-based, no filesystem I/O | | `test_doctor_fleet_checks.py` | Tests for fleet doctor checks — Group M ambient env/infra/campaign, Group N feature gates and registry | | `test_doctor_migration.py` | Tests for doctor quota cache schema, install classification, version consistency, and drift | From 406d1406f1fca59e0ac455356ee30445b43be314 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:06:51 -0700 Subject: [PATCH 4/7] fix: use AGENT_BACKEND_CODEX constant instead of string literal; bump doctor check count to 40 --- src/autoskillit/cli/doctor/_doctor_runtime.py | 10 ++++++++-- tests/docs/test_doc_counts.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index 1098dedf43..f746e501da 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -8,7 +8,13 @@ import regex as re -from autoskillit.core import CodingAgentBackend, Severity, default_log_dir, get_logger +from autoskillit.core import ( + AGENT_BACKEND_CODEX, + CodingAgentBackend, + Severity, + default_log_dir, + get_logger, +) from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION from ._doctor_types import DoctorResult @@ -243,7 +249,7 @@ def _check_codex_graduation( entry = json.loads(line) except (json.JSONDecodeError, ValueError): continue - if entry.get("backend") == "codex": + if entry.get("backend") == AGENT_BACKEND_CODEX: smoke_status = "pass" if entry.get("success") else "fail" break except OSError: diff --git a/tests/docs/test_doc_counts.py b/tests/docs/test_doc_counts.py index dddeb38690..3181494561 100644 --- a/tests/docs/test_doc_counts.py +++ b/tests/docs/test_doc_counts.py @@ -238,14 +238,14 @@ def test_quota_thresholds_defaults() -> None: def test_doctor_check_count_is_31() -> None: - # 38 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) + # 39 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) # + 4 ambient env (18–21) + 2 feature (22–23) + 6 gated franchise (24–29) - # + 1 codex version (30) + 1 script binary (31). + # + 1 codex version (30) + 1 script binary (31) + 1 codex graduation (33). # The docs claim 17 user-visible checks; the gap is intentional (Check 2/4/7 # split into sub-markers here but appear as single entries in docs). # Update both tests whenever a new doctor check is added. count = _count_doctor_checks() - assert count == 39, f"Expected 39 doctor checks; found {count}" + assert count == 40, f"Expected 40 doctor checks; found {count}" def test_bundled_recipe_count_is_15() -> None: From b8ac7e299fbf6ff93dae467514c583194c06519c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:09:51 -0700 Subject: [PATCH 5/7] fix: derive backend log filter from backend.name instead of comparing AGENT_BACKEND_CODEX constant --- src/autoskillit/cli/doctor/_doctor_runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index f746e501da..5facd5a0f7 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -9,7 +9,6 @@ import regex as re from autoskillit.core import ( - AGENT_BACKEND_CODEX, CodingAgentBackend, Severity, default_log_dir, @@ -210,6 +209,7 @@ def _check_codex_graduation( if backend is None or not backend.capabilities.version_check_command: return DoctorResult(Severity.OK, check_name, "Skipped (no codex backend)") + backend_name = backend.name log_root = log_dir or default_log_dir() # Criterion 1: version check @@ -249,7 +249,7 @@ def _check_codex_graduation( entry = json.loads(line) except (json.JSONDecodeError, ValueError): continue - if entry.get("backend") == AGENT_BACKEND_CODEX: + if entry.get("backend") == backend_name: smoke_status = "pass" if entry.get("success") else "fail" break except OSError: From 6e4daf267e0fbfb2ffd43ee91d4bd578f1352510 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:37:57 -0700 Subject: [PATCH 6/7] fix(review): rename stale test function and update check count comment to 40 Function was named test_doctor_check_count_is_31 while asserting count == 40. Comment said "39 total" but Check 32 (_check_codex_mcp_timeouts) was missing from the breakdown, making the arithmetic off by one. Co-Authored-By: Claude Sonnet 4.6 --- tests/docs/test_doc_counts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/docs/test_doc_counts.py b/tests/docs/test_doc_counts.py index 3181494561..a46ff99f92 100644 --- a/tests/docs/test_doc_counts.py +++ b/tests/docs/test_doc_counts.py @@ -237,10 +237,11 @@ def test_quota_thresholds_defaults() -> None: assert long_ == pytest.approx(95.0) -def test_doctor_check_count_is_31() -> None: - # 39 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) +def test_doctor_check_count_is_40() -> None: + # 40 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) # + 4 ambient env (18–21) + 2 feature (22–23) + 6 gated franchise (24–29) - # + 1 codex version (30) + 1 script binary (31) + 1 codex graduation (33). + # + 1 codex version (30) + 1 script binary (31) + 1 MCP timeouts (32) + # + 1 codex graduation (33). # The docs claim 17 user-visible checks; the gap is intentional (Check 2/4/7 # split into sub-markers here but appear as single entries in docs). # Update both tests whenever a new doctor check is added. From d038b8bffd0a9930dd52338165bd8298561ba9e5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 27 Jun 2026 07:38:10 -0700 Subject: [PATCH 7/7] fix(review): guard probe-cache entry access with isinstance check any(e.get("passed") ...) raises AttributeError if a cache entry value is a scalar (str, int, None). Added isinstance(e, dict) guard to make the assumption explicit and avoid uncaught AttributeError. Co-Authored-By: Claude Sonnet 4.6 --- src/autoskillit/cli/doctor/_doctor_runtime.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index 5facd5a0f7..74cc3e98fb 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -223,7 +223,11 @@ def _check_codex_graduation( raw = json.loads(probe_path.read_text()) entries = raw.get("entries", {}) if isinstance(raw, dict) else {} if entries: - probe_status = "pass" if any(e.get("passed") for e in entries.values()) else "fail" + probe_status = ( + "pass" + if any(isinstance(e, dict) and e.get("passed") for e in entries.values()) + else "fail" + ) except (OSError, json.JSONDecodeError, ValueError): pass