diff --git a/src/autoskillit/cli/doctor/AGENTS.md b/src/autoskillit/cli/doctor/AGENTS.md index 1fe02444db..1598899c06 100644 --- a/src/autoskillit/cli/doctor/AGENTS.md +++ b/src/autoskillit/cli/doctor/AGENTS.md @@ -1,6 +1,6 @@ # doctor/ -Diagnostic health checks for the autoskillit installation (28 checks). +Diagnostic health checks for the autoskillit installation (33 checks). ## Files @@ -15,7 +15,7 @@ Diagnostic health checks for the autoskillit installation (28 checks). | `_doctor_hooks.py` | Hook registration, executability, and registry drift checks | | `_doctor_install.py` | Install path, entry points, version drift, update dismissal checks | | `_doctor_mcp.py` | MCP server registration, dual registration, plugin cache checks | -| `_doctor_runtime.py` | Quota cache schema version, claude process state, and codex version checks | +| `_doctor_runtime.py` | Quota cache schema version, claude process state, codex version, and codex graduation checks | ## Architecture Notes diff --git a/src/autoskillit/cli/doctor/__init__.py b/src/autoskillit/cli/doctor/__init__.py index eeca2a1250..fd293adf2d 100644 --- a/src/autoskillit/cli/doctor/__init__.py +++ b/src/autoskillit/cli/doctor/__init__.py @@ -61,6 +61,7 @@ ) from ._doctor_runtime import ( _check_claude_process_state_breakdown, + _check_codex_graduation, _check_codex_version, _check_quota_cache_schema, _check_script_binary, @@ -213,6 +214,9 @@ def run_doctor(*, output_json: bool = False) -> None: _check_codex_mcp_timeouts(backend=_backend, run_skill=cfg.run_skill, fleet=cfg.fleet) ) + # Check 33: Codex graduation criteria + results.append(_check_codex_graduation(backend=_backend)) + # Output if output_json: print( diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index f31dbd544c..74cc3e98fb 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -8,7 +8,12 @@ import regex as re -from autoskillit.core import CodingAgentBackend, Severity, get_logger +from autoskillit.core import ( + CodingAgentBackend, + Severity, + default_log_dir, + get_logger, +) from autoskillit.execution import QUOTA_CACHE_SCHEMA_VERSION from ._doctor_types import DoctorResult @@ -192,3 +197,76 @@ def _check_script_binary() -> DoctorResult: "script(1) present but -qefc flags unsupported — PTY wrapping may not work correctly", ) return DoctorResult(Severity.OK, check_name, "script(1) available with -qefc support") + + +def _check_codex_graduation( + *, + backend: CodingAgentBackend | None = None, + log_dir: Path | None = None, +) -> DoctorResult: + check_name = "codex_graduation" + + if backend is None or not backend.capabilities.version_check_command: + return DoctorResult(Severity.OK, check_name, "Skipped (no codex backend)") + + backend_name = backend.name + log_root = log_dir or default_log_dir() + + # Criterion 1: version check + version_result = _check_codex_version(backend=backend) + version_status = "pass" if version_result.severity == Severity.OK else "fail" + + # Criterion 2: probe-harness cache + probe_path = log_root / "codex-probe-cache.json" + probe_status = "not-yet-run" + try: + raw = json.loads(probe_path.read_text()) + entries = raw.get("entries", {}) if isinstance(raw, dict) else {} + if entries: + probe_status = ( + "pass" + if any(isinstance(e, dict) and e.get("passed") for e in entries.values()) + else "fail" + ) + except (OSError, json.JSONDecodeError, ValueError): + pass + + # Criterion 3: matrix last-run result + matrix_path = log_root / "codex-matrix-result.json" + matrix_status = "not-yet-run" + try: + matrix_raw = json.loads(matrix_path.read_text()) + if isinstance(matrix_raw, dict) and "passed" in matrix_raw: + matrix_status = "pass" if matrix_raw["passed"] else "fail" + except (OSError, json.JSONDecodeError, ValueError): + pass + + # Criterion 4: sessions.jsonl smoke + sessions_path = log_root / "sessions.jsonl" + smoke_status = "not-found" + try: + for line in reversed(sessions_path.read_text().splitlines()): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if entry.get("backend") == backend_name: + smoke_status = "pass" if entry.get("success") else "fail" + break + except OSError: + pass + + statuses = [version_status, probe_status, matrix_status, smoke_status] + summary = ( + f"version={version_status} | probe={probe_status}" + f" | matrix={matrix_status} | smoke={smoke_status}" + ) + + if not all(s == "pass" for s in statuses): + pending = sum(1 for s in statuses if s != "pass") + summary += f" — EXPERIMENTAL hold: {pending} of 4 criteria pending" + + return DoctorResult(Severity.INFO, check_name, summary) diff --git a/tests/cli/AGENTS.md b/tests/cli/AGENTS.md index d0d8298361..23172ab849 100644 --- a/tests/cli/AGENTS.md +++ b/tests/cli/AGENTS.md @@ -28,7 +28,7 @@ CLI command, subcommand, and interactive workflow tests. | `test_cook_order_prompt.py` | Tests: cook CLI order command — system prompt content, MCP prefix selection, ownership | | `test_cook_workspace.py` | Tests: cook CLI workspace init and clean commands | | `test_doctor.py` | Tests for CLI doctor command and related utilities | -| `test_doctor_backend_guards.py` | Tests for doctor backend guard checks (stale MCP, MCP registered, process state) and run_doctor backend wiring | +| `test_doctor_backend_guards.py` | Tests for doctor backend guard checks (stale MCP, MCP registered, process state, codex graduation) and run_doctor backend wiring | | `test_doctor_codex_mcp.py` | Tests for _check_mcp_server_registered Codex branch — monkeypatch-based, no filesystem I/O | | `test_doctor_fleet_checks.py` | Tests for fleet doctor checks — Group M ambient env/infra/campaign, Group N feature gates and registry | | `test_doctor_migration.py` | Tests for doctor quota cache schema, install classification, version consistency, and drift | diff --git a/tests/cli/test_doctor_backend_guards.py b/tests/cli/test_doctor_backend_guards.py index 4df90dd59d..32bfa00920 100644 --- a/tests/cli/test_doctor_backend_guards.py +++ b/tests/cli/test_doctor_backend_guards.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace @@ -579,3 +580,114 @@ def test_version_above_minimum_returns_ok(self, monkeypatch: pytest.MonkeyPatch) result = _check_codex_version(backend=CodexBackend()) assert result.severity == Severity.OK assert "0.131.0" in result.message + + +class TestCheckCodexGraduation: + """Tests for _check_codex_graduation doctor check (Check 33).""" + + def _codex_result(self, stdout: str, returncode: int = 0, stderr: str = ""): + return type( + "CompletedProcess", + (), + {"returncode": returncode, "stdout": stdout, "stderr": stderr}, + )() + + def test_skip_for_none_backend(self) -> None: + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + + result = _check_codex_graduation(backend=None) + assert result.severity == Severity.OK + assert result.check == "codex_graduation" + assert "skipped" in result.message.lower() + + def test_skip_for_non_codex_backend(self) -> None: + from types import SimpleNamespace + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + + stub = SimpleNamespace(capabilities=SimpleNamespace(version_check_command="")) + result = _check_codex_graduation(backend=stub) + assert result.severity == Severity.OK + assert "skipped" in result.message.lower() + + def test_all_green_omits_hold_note( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + from datetime import UTC, datetime + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + ts = datetime.now(UTC).isoformat() + (tmp_path / "codex-probe-cache.json").write_text( + json.dumps( + { + "schema_version": 1, + "entries": {"0.130.0": {"passed": True, "probe_timestamp": ts}}, + } + ) + ) + (tmp_path / "codex-matrix-result.json").write_text(json.dumps({"passed": True})) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"backend": "codex", "success": True}) + "\n" + ) + + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "version=pass" in result.message + assert "probe=pass" in result.message + assert "matrix=pass" in result.message + assert "smoke=pass" in result.message + assert "EXPERIMENTAL" not in result.message + + def test_non_green_includes_hold_note( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + (tmp_path / "sessions.jsonl").write_text( + json.dumps({"backend": "codex", "success": True}) + "\n" + ) + + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "not-yet-run" in result.message + assert "EXPERIMENTAL" in result.message + + def test_absent_files_yield_not_yet_run( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import subprocess + + from autoskillit.cli.doctor._doctor_runtime import _check_codex_graduation + from autoskillit.core import Severity + from autoskillit.execution.backends.codex import CodexBackend + + monkeypatch.setattr( + subprocess, + "run", + lambda *a, **kw: self._codex_result("Codex 0.130.0\n"), + ) + result = _check_codex_graduation(backend=CodexBackend(), log_dir=tmp_path) + assert result.severity == Severity.INFO + assert "probe=not-yet-run" in result.message + assert "matrix=not-yet-run" in result.message + assert "smoke=not-found" in result.message diff --git a/tests/docs/test_doc_counts.py b/tests/docs/test_doc_counts.py index dddeb38690..a46ff99f92 100644 --- a/tests/docs/test_doc_counts.py +++ b/tests/docs/test_doc_counts.py @@ -237,15 +237,16 @@ def test_quota_thresholds_defaults() -> None: assert long_ == pytest.approx(95.0) -def test_doctor_check_count_is_31() -> None: - # 38 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) +def test_doctor_check_count_is_40() -> None: + # 40 total = 17 numbered base + 7 lettered sub-checks (2b–2e, 4b, 7b, 7c) # + 4 ambient env (18–21) + 2 feature (22–23) + 6 gated franchise (24–29) - # + 1 codex version (30) + 1 script binary (31). + # + 1 codex version (30) + 1 script binary (31) + 1 MCP timeouts (32) + # + 1 codex graduation (33). # The docs claim 17 user-visible checks; the gap is intentional (Check 2/4/7 # split into sub-markers here but appear as single entries in docs). # Update both tests whenever a new doctor check is added. count = _count_doctor_checks() - assert count == 39, f"Expected 39 doctor checks; found {count}" + assert count == 40, f"Expected 40 doctor checks; found {count}" def test_bundled_recipe_count_is_15() -> None: