Skip to content

Commit c552362

Browse files
Trecekclaude
andauthored
T5-P5-A8-WP1 Provide the Reusable Canary State Machine and GitHub Issue Updater (#4107)
## Summary Create two new private modules that provide the reusable building blocks for live probe classes: 1. **`src/autoskillit/_probe_canary.py`** — Package-root private module containing `ErrorKind` (StrEnum), `N_CONSECUTIVE_FLAKE_GUARD` constant, `CanaryState` dataclass (JSON-persisted state machine with failure streak tracking), and `CanaryIssueUpdater` (GitHub issue ensure/update via `run_gh` from `autoskillit.core`). 2. **`src/autoskillit/execution/backends/_probe_cache.py`** — Probe result caching with `ProbeResult` frozen dataclass, 24-hour TTL, and versioned-JSON read/write functions using `autoskillit.core` IO primitives. Both modules are IL-0 compatible (only stdlib + `autoskillit.core` imports). `run_gh` is exported from `autoskillit.core` (via `core/_cmd_runner.py`), confirmed at `core/__init__.pyi:5`. Closes #4041 ## Implementation Plan Plan file: `/home/talon/projects/autoskillit-runs/impl-20260613-041038-627697/.autoskillit/temp/make-plan/t5_p5_a8_wp1_canary_state_machine_plan_2026-06-13_041300.md` 🤖 Generated with [Claude Code](https://claude.com/claude-code) via AutoSkillit <!-- autoskillit:pipeline-signature steps=prepare_pr,run_arch_lenses,compose_pr,annotate_pr_diff,review_pr --> ## Token Usage Summary | Step | Model | count | uncached | output | cache_read | peak_ctx | turns | cache_write | time | |------|-------|-------|----------|--------|------------|----------|-------|-------------|------| | plan* | opus[1m] | 1 | 2.5k | 33.9k | 1.7M | 111.0k | 59 | 99.7k | 16m 58s | | verify* | sonnet | 1 | 1.0k | 10.3k | 310.0k | 63.7k | 18 | 42.9k | 5m 52s | | implement* | MiniMax-M3 | 1 | 3.8M | 17.5k | 0 | 0 | 109 | 0 | 16m 33s | | audit_impl* | sonnet | 1 | 849 | 14.2k | 213.5k | 49.7k | 16 | 44.4k | 5m 4s | | prepare_pr* | MiniMax-M3 | 2 | 497.1k | 5.8k | 0 | 0 | 29 | 0 | 2m 0s | | compose_pr* | MiniMax-M3 | 1 | 213.3k | 1.5k | 0 | 0 | 13 | 0 | 43s | | **Total** | | | 4.5M | 83.1k | 2.2M | 111.0k | | 187.0k | 47m 12s | \* *Step used a non-Anthropic provider; caching behavior may differ.* ## Token Efficiency | Step | LoC Changed | cache_read/LoC | cache_write/LoC | output/LoC | |------|-------------|----------------|-----------------|------------| | plan | 0 | — | — | — | | verify | 0 | — | — | — | | implement | 554 | 0.0 | 0.0 | 31.6 | | audit_impl | 0 | — | — | — | | prepare_pr | 0 | — | — | — | | compose_pr | 0 | — | — | — | | **Total** | **554** | 4057.4 | 337.5 | 150.0 | ## Model Usage Breakdown | Model | steps | uncached | output | cache_read | cache_write | time | |-------|-------|----------|--------|------------|-------------|------| | opus[1m] | 1 | 2.5k | 33.9k | 1.7M | 99.7k | 16m 58s | | sonnet | 2 | 1.9k | 24.5k | 523.5k | 87.3k | 10m 56s | | MiniMax-M3 | 3 | 4.5M | 24.7k | 0 | 0 | 19m 17s | --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 5750a52 commit c552362

11 files changed

Lines changed: 590 additions & 2 deletions

File tree

.autoskillit/test-source-map.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15071,5 +15071,11 @@
1507115071
"tests/server/test_tools_git.py",
1507215072
"tests/server/test_tools_git_merge_cleanup.py",
1507315073
"tests/workspace/test_worktree.py"
15074+
],
15075+
"src/autoskillit/_probe_canary.py": [
15076+
"tests/execution/backends/test_probe_canary.py"
15077+
],
15078+
"src/autoskillit/execution/backends/_probe_cache.py": [
15079+
"tests/execution/backends/test_probe_cache.py"
1507415080
]
1507515081
}

src/autoskillit/AGENTS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Package root — entry points, hook registry, and cross-cutting utilities.
1111
| `_llm_triage.py` | Contract staleness triage (Haiku subprocess) |
1212
| `smoke_utils/` | Callables for smoke-test pipeline `run_python` steps (package) |
1313
| `hook_registry.py` | `HookDef`, `HOOK_REGISTRY`, `generate_hooks_json` |
14+
| `_probe_canary.py` | Canary state machine and GitHub issue updater for live probes (IL-1) |
1415
| `_test_filter.py` | Test filter manifest: glob-to-test-directory mapping |
1516
| `version.py` | Version health utilities (IL-0) |
1617

src/autoskillit/_probe_canary.py

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
"""Reusable canary state machine and GitHub issue updater for live probes.
2+
3+
IL-1 module: imports only stdlib and `autoskillit.core`. Provides the
4+
persistence + flake-guard primitives that live probe classes build on.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
import json
10+
import os
11+
import subprocess
12+
import tempfile
13+
from dataclasses import asdict, dataclass
14+
from enum import StrEnum, unique
15+
from pathlib import Path
16+
17+
from autoskillit.core import atomic_write, get_logger, run_gh
18+
19+
logger = get_logger(__name__)
20+
21+
N_CONSECUTIVE_FLAKE_GUARD: int = 3
22+
23+
24+
@unique
25+
class ErrorKind(StrEnum):
26+
NETWORK = "network"
27+
SCHEMA = "schema"
28+
29+
30+
@dataclass
31+
class CanaryState:
32+
network_streak: int = 0
33+
schema_streak: int = 0
34+
last_issue_number: int | None = None
35+
36+
@classmethod
37+
def load(cls, path: Path) -> CanaryState:
38+
try:
39+
raw = json.loads(path.read_text(encoding="utf-8"))
40+
except (FileNotFoundError, json.JSONDecodeError, OSError):
41+
return cls()
42+
if not isinstance(raw, dict):
43+
return cls()
44+
return cls(
45+
network_streak=raw.get("network_streak", 0),
46+
schema_streak=raw.get("schema_streak", 0),
47+
last_issue_number=raw.get("last_issue_number"),
48+
)
49+
50+
def save(self, path: Path) -> None:
51+
atomic_write(path, json.dumps(asdict(self), indent=2))
52+
53+
def record_failure(self, kind: ErrorKind) -> None:
54+
if kind is ErrorKind.NETWORK:
55+
self.network_streak += 1
56+
elif kind is ErrorKind.SCHEMA:
57+
self.schema_streak += 1
58+
else:
59+
raise ValueError(f"Unhandled ErrorKind: {kind!r}")
60+
61+
def record_success(self) -> None:
62+
self.network_streak = 0
63+
self.schema_streak = 0
64+
65+
def should_report(self, flake_guard: int = N_CONSECUTIVE_FLAKE_GUARD) -> bool:
66+
return self.network_streak >= flake_guard or self.schema_streak >= flake_guard
67+
68+
69+
def _run_gh_with_body_file(args: list[str], body: str) -> subprocess.CompletedProcess[str]:
70+
with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f:
71+
f.write(body)
72+
body_path = f.name
73+
try:
74+
return run_gh([*args, "--body-file", body_path])
75+
finally:
76+
try:
77+
os.unlink(body_path)
78+
except OSError:
79+
logger.debug("canary_body_file_unlink_failed", path=body_path)
80+
81+
82+
class CanaryIssueUpdater:
83+
def __init__(self, *, owner: str, repo: str) -> None:
84+
self._owner = owner
85+
self._repo = repo
86+
87+
def ensure_issue(self, state: CanaryState, title: str, body: str) -> int:
88+
existing = self._find_existing(title)
89+
if existing is not None:
90+
result = _run_gh_with_body_file(
91+
[
92+
"issue",
93+
"edit",
94+
str(existing),
95+
"--repo",
96+
f"{self._owner}/{self._repo}",
97+
],
98+
body,
99+
)
100+
if result.returncode != 0:
101+
logger.warning(
102+
"canary_issue_edit_failed",
103+
issue=existing,
104+
stderr=result.stderr,
105+
)
106+
state.last_issue_number = existing
107+
return existing
108+
result = _run_gh_with_body_file(
109+
[
110+
"issue",
111+
"create",
112+
"--repo",
113+
f"{self._owner}/{self._repo}",
114+
"--title",
115+
title,
116+
"--json",
117+
"number",
118+
],
119+
body,
120+
)
121+
if result.returncode != 0:
122+
msg = f"gh issue create failed: {result.stderr}"
123+
raise RuntimeError(msg)
124+
try:
125+
issue_number = json.loads(result.stdout)["number"]
126+
except (json.JSONDecodeError, KeyError) as exc:
127+
msg = f"gh issue create returned unexpected output: {result.stdout!r}"
128+
raise RuntimeError(msg) from exc
129+
state.last_issue_number = issue_number
130+
return issue_number
131+
132+
def _find_existing(self, title: str) -> int | None:
133+
result = run_gh(
134+
[
135+
"issue",
136+
"list",
137+
"--repo",
138+
f"{self._owner}/{self._repo}",
139+
"--search",
140+
title,
141+
"--state",
142+
"open",
143+
"--json",
144+
"number,title",
145+
"--limit",
146+
"10",
147+
],
148+
)
149+
if result.returncode != 0:
150+
logger.warning(
151+
"canary_find_existing_failed",
152+
returncode=result.returncode,
153+
stderr=result.stderr,
154+
)
155+
return None
156+
try:
157+
issues = json.loads(result.stdout)
158+
except json.JSONDecodeError:
159+
return None
160+
if not isinstance(issues, list):
161+
return None
162+
for issue in issues:
163+
if issue.get("title") == title:
164+
number = issue.get("number")
165+
if isinstance(number, int):
166+
return number
167+
return None

src/autoskillit/execution/backends/AGENTS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ IL-1 backend abstraction layer — concrete `CodingAgentBackend` implementations
99
| `__init__.py` | `BACKEND_REGISTRY`, `get_backend()` factory, re-exports |
1010
| `_backend_cmd_builder_base.py` | `BackendCmdBuilderBase` ABC (frozen dataclass), `FlagVocabulary` NamedTuple, `SHARED_BASELINE_ENV` — canonical location for shared env-assembly keys |
1111
| `_composite_locator.py` | `CompositeSessionLocator` — single dispatch point iterating BACKEND_REGISTRY for session location |
12+
| `_probe_cache.py` | `ProbeResult`, `PROBE_CACHE_TTL`, `read_probe_cache`, `write_probe_cache` — versioned probe result cache |
1213
| `claude.py` | `ClaudeCodeBackend` (incl. `validate_session_layout`), `ClaudeEnvPolicy`, `ClaudeSessionLocator`, `ClaudeStreamParser`, `ClaudeResultParser` (prompt utilities moved to `_claude_prompt.py`) |
1314
| `_claude_prompt.py` | Prompt injection utilities, session constants (`_ensure_skill_prefix`, `_inject_completion_directive`, `_compose_resume_prompt`, etc.), shared by claude + codex + commands |
1415
| `codex.py` | `CodexFlags`, `CodexBackend` (incl. `validate_session_layout`, `setup_session_dir` agent TOML generation via `_generate_agent_tomls`), `CodexEnvPolicy`, `CodexSessionLocator` (parse/config moved to `_codex_parse.py` / `_codex_config.py`) |
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""Versioned probe result cache with 24-hour TTL.
2+
3+
IL-1 module: imports only stdlib and `autoskillit.core`. Stores a per-cli-version
4+
`ProbeResult` keyed by `cli_version`, surviving across runs while staying
5+
under `PROBE_CACHE_TTL`.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from dataclasses import dataclass
11+
from datetime import UTC, datetime, timedelta
12+
from pathlib import Path
13+
14+
from autoskillit.core import get_logger, read_versioned_json, write_versioned_json
15+
16+
logger = get_logger(__name__)
17+
18+
PROBE_CACHE_TTL: timedelta = timedelta(hours=24)
19+
_SCHEMA_VERSION: int = 1
20+
21+
22+
@dataclass(frozen=True, slots=True)
23+
class ProbeResult:
24+
cli_version: str
25+
passed: bool
26+
failure_detail: str | None
27+
probe_timestamp: str
28+
29+
30+
def read_probe_cache(cache_path: Path, cli_version: str) -> ProbeResult | None:
31+
raw = read_versioned_json(cache_path, _SCHEMA_VERSION, logger=logger)
32+
if raw is None:
33+
return None
34+
entries: dict[str, dict] = raw.get("entries", {})
35+
entry = entries.get(cli_version)
36+
if entry is None:
37+
return None
38+
try:
39+
ts = datetime.fromisoformat(entry["probe_timestamp"])
40+
if (datetime.now(UTC) - ts) > PROBE_CACHE_TTL:
41+
return None
42+
except (KeyError, ValueError, TypeError):
43+
return None
44+
return ProbeResult(
45+
cli_version=cli_version,
46+
passed=entry.get("passed", False),
47+
failure_detail=entry.get("failure_detail"),
48+
probe_timestamp=entry["probe_timestamp"],
49+
)
50+
51+
52+
def write_probe_cache(cache_path: Path, result: ProbeResult) -> None:
53+
try:
54+
existing = read_versioned_json(cache_path, _SCHEMA_VERSION, logger=logger)
55+
entries: dict[str, dict] = (existing or {}).get("entries", {})
56+
entries[result.cli_version] = {
57+
"passed": result.passed,
58+
"failure_detail": result.failure_detail,
59+
"probe_timestamp": result.probe_timestamp,
60+
}
61+
write_versioned_json(cache_path, {"entries": entries}, _SCHEMA_VERSION)
62+
except OSError:
63+
logger.debug("probe_cache_write_failed", path=str(cache_path), exc_info=True)

tests/_test_filter.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ class ImportContext(enum.StrEnum):
168168
}
169169

170170
MODULE_CASCADE_CORE: dict[str, frozenset[str]] = {
171-
"_cmd_runner": frozenset({"cli", "core", "recipe"}),
171+
"_cmd_runner": frozenset({"cli", "core", "recipe", "_probe_canary"}),
172172
"_json": frozenset({"core", "execution", "pipeline", "recipe", "server"}),
173173
"feature_flags": frozenset(
174174
{"core", "cli", "config", "execution", "recipe", "server", "workspace"}
@@ -605,6 +605,7 @@ class ImportContext(enum.StrEnum):
605605
"hook_registry",
606606
"planner",
607607
"smoke_utils",
608+
"_probe_canary",
608609
}
609610
),
610611
# L1
@@ -896,6 +897,7 @@ class ImportContext(enum.StrEnum):
896897
"_test_filter": frozenset({"arch", "infra", "contracts"}),
897898
"smoke_utils": frozenset({"test_smoke_utils.py", "recipe", "smoke_utils"}),
898899
"version": frozenset({"test_version.py", "server", "cli"}),
900+
"_probe_canary": frozenset({"core", "execution/backends/test_probe_canary.py"}),
899901
}
900902

901903
# ---------------------------------------------------------------------------
@@ -979,6 +981,7 @@ class ImportContext(enum.StrEnum):
979981
"version": frozenset({"test_version.py"}),
980982
"_test_filter": frozenset({"arch", "contracts"}),
981983
"report": frozenset({"report"}),
984+
"_probe_canary": frozenset({"core", "execution/backends/test_probe_canary.py"}),
982985
}
983986

984987
# ---------------------------------------------------------------------------

tests/arch/test_subpackage_isolation.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ def _get_call_func_name(node: ast.Call) -> str | None:
7777
"validator", # recipe/validator.py: defensive exemption for decorator-based rule registry
7878
"settings", # config/settings.py: _CONFIG_SCHEMA = _build_config_schema()
7979
"_headless_path_tokens", # execution/_headless_path_tokens.py: _OUTPUT_PATH_TOKENS
80+
"_probe_cache", # execution/backends/_probe_cache.py: PROBE_CACHE_TTL = timedelta(...)
8081
# _STABLE_DISMISS_WINDOW = timedelta(days=7), _DEV_DISMISS_WINDOW = timedelta(hours=12)
8182
"_install_info", # cli/_install_info.py: window constants (see comment above)
8283
# KITCHEN_GUARDED_COMMANDS: frozenset[str]
@@ -881,7 +882,7 @@ def test_no_subpackage_exceeds_10_files() -> None:
881882
"recipe/rules": 51, # +commit_guard_regression_route +rules_model +rules_gitignored_deliverable # noqa: E501
882883
"server/tools": 27, # +_preflight.py (dispatch-feasibility preflight)
883884
"hooks/guards": 32, # +fleet_claim_guard, +reset_resume_gate, +recipe_read_guard
884-
"execution/backends": 11, # +_composite_locator.py (CompositeSessionLocator)
885+
"execution/backends": 12, # +_composite_locator.py, +_probe_cache.py
885886
}
886887
violations: list[str] = []
887888
dirs_to_check: list[Path] = []

tests/contracts/test_package_gateways.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,7 @@ def test_root_module_allowlist() -> None:
397397
"__init__.py",
398398
"__main__.py",
399399
"_llm_triage.py",
400+
"_probe_canary.py",
400401
"_test_filter.py",
401402
"hook_registry.py",
402403
"version.py",

0 commit comments

Comments
 (0)