Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/autoskillit/core/types/_type_constants_registries.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,14 @@ def required_backends(self) -> frozenset[str]:
return frozenset()


# Semantics divergence: fix-required has different enforcement in HOOK_REGISTRY vs
# SKILL_CAPABILITY_REGISTRY. In HOOK_REGISTRY, fix-required triggers _check_backend_compat
# to block dispatch on backends whose applicable_guards don't cover the hook's scripts.
# In SKILL_CAPABILITY_REGISTRY, fix-required is advisory only — it does NOT block dispatch
# (required_backends returns frozenset() for all non-not-applicable statuses). Skills
# declaring fix-required capabilities (agent_subagent, agent_model, cross_skill_ref) are
# still Codex-dispatchable because the capability is documentary about the feature
# being incomplete, not a hard blocker. Do not conflate the two registries' semantics.
SKILL_CAPABILITY_REGISTRY: dict[str, SkillCapabilityDef] = {
"agent_subagent": SkillCapabilityDef(
description="Agent(subagent_type=...) tool — delegates to specialized subagent",
Expand Down
40 changes: 40 additions & 0 deletions src/autoskillit/server/_lifespan.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
session_type as _resolve_session_type,
)
from autoskillit.execution import (
BACKEND_REGISTRY,
RecordingSubprocessRunner,
ensure_codex_mcp_registered,
)
Expand All @@ -54,6 +55,7 @@
sweep_stale_dispatch_labels,
)
from autoskillit.hook_registry import (
HOOK_REGISTRY,
HOOK_REGISTRY_HASH,
find_broken_hook_scripts,
generate_hooks_json,
Expand Down Expand Up @@ -126,6 +128,42 @@ def run_startup_hook_health_check() -> list[str]:
return []


def run_startup_fix_required_coverage_check() -> None:
"""Validate that fix-required hook script stems are covered by at least one backend.

The dispatch gate in tools_execution._check_backend_compat refuses all skill
dispatches on a backend if HOOK_REGISTRY contains fix-required hooks whose
script stems are not in that backend's applicable_guards. This check provides
defense-in-depth: if the cross-registry invariant is violated, the server
fails to start rather than accepting requests it will later crash on.

Raises RuntimeError if any fix-required hook's script stems are not covered
by the union of all registered backends' applicable_guards. A fix-required
hook that IS covered by at least one backend is valid and does not raise.
"""
all_guards: set[str] = set()
for cls in BACKEND_REGISTRY.values():
try:
all_guards.update(cls().capabilities.applicable_guards)
except Exception as exc:
raise RuntimeError(
f"Backend {cls.__name__!r} constructor raised during startup "
f"fix-required coverage check: {exc}"
) from exc
for h in HOOK_REGISTRY:
if h.codex_status != "fix-required":
continue
stems = frozenset(Path(s).stem for s in h.scripts) if h.scripts else frozenset()
if stems and not stems.issubset(all_guards):
missing = sorted(stems - all_guards)
raise RuntimeError(
f"HOOK_REGISTRY fix-required entry (matcher={h.matcher!r}) has "
f"guard scripts {missing} not covered by any backend's "
f"applicable_guards. This will brick dispatch for backends "
f"missing these guards."
)


def _finalize_recorder() -> None:
"""Finalize the recording subprocess runner if one is active."""
ctx = _get_ctx_or_none()
Expand Down Expand Up @@ -529,6 +567,8 @@ async def _autoskillit_lifespan(server: Any) -> Any:
try:
from autoskillit.server import _state # circular-break

run_startup_fix_required_coverage_check()

event = _asyncio.Event()
_state._startup_ready = event
write_readiness_sentinel()
Expand Down
1 change: 1 addition & 0 deletions tests/arch/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ AST enforcement, sub-package layer contracts, and architectural invariant tests.
| `test_canonical_constant_consumption.py` | Architectural invariant: every *_ENV_FORWARD_VARS constant must have a production consumer |
| `test_backend_name_sync.py` | Architectural invariant: KNOWN_BACKEND_NAMES (IL-0) must match BACKEND_REGISTRY keys (IL-1) |
| `test_capability_consistency.py` | Behavioral arch tests: BackendCapabilities filesystem consistency — applicable guards exist on disk, required session files are created, session_dir_symlinks entries are symlinks |
| `test_cross_registry_dispatch_sufficiency.py` | Cross-registry dispatch sufficiency: real HOOK_REGISTRY × real BackendCapabilities — detects fix-required hooks that would brick backends with insufficient applicable_guards |
| `test_capability_consumption.py` | Architectural invariant: every BackendCapabilities field must be consumed in production |
| `test_capability_docstrings.py` | Architectural invariant: BackendCapabilities must have class and field documentation |
| `test_no_backend_name_bypass.py` | Architectural invariant: backend-specific behavior must use capability fields, not name comparisons |
Expand Down
91 changes: 91 additions & 0 deletions tests/arch/test_cross_registry_dispatch_sufficiency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""Cross-registry dispatch sufficiency: real HOOK_REGISTRY × real BackendCapabilities.

Regression suite for issue #4082 — guards against a class of bug where HOOK_REGISTRY
fix-required entries silently brick dispatch on backends whose applicable_guards do
not cover the hook's script stems. The dispatch gate in tools_execution._check_backend_compat
correctly refuses dispatch in this state, but the bug class had no test that crossed the
HOOK_REGISTRY ↔ BackendCapabilities boundary until now.

These tests exercise the real registries through the real gate logic, not synthetic
monkeypatched copies. This is the defense-in-depth layer that catches reclassifications
which would brick dispatch silently.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from autoskillit.execution.backends import BACKEND_REGISTRY
from autoskillit.hook_registry import HOOK_REGISTRY
from autoskillit.server.tools.tools_execution import _get_fix_required_hook_matchers

pytestmark = [pytest.mark.layer("arch"), pytest.mark.small]


def _collect_fix_required_stems() -> set[str]:
"""Extract unique script stems from all fix-required hooks in HOOK_REGISTRY."""
stems: set[str] = set()
for h in HOOK_REGISTRY:
if h.codex_status == "fix-required":
stems.update(Path(s).stem for s in h.scripts)
return stems


def _all_fix_required_matchers() -> list[str]:
"""Extract matchers from all fix-required hooks in HOOK_REGISTRY."""
return [h.matcher for h in HOOK_REGISTRY if h.codex_status == "fix-required"]


@pytest.mark.parametrize("backend_name,backend_cls", sorted(BACKEND_REGISTRY.items()))
def test_no_backend_bricked_by_fix_required_hooks(backend_name: str, backend_cls: type) -> None:
"""Every backend's applicable_guards must cover all fix-required hook scripts.

Runs the exact same function the dispatch gate uses — _get_fix_required_hook_matchers —
against the real HOOK_REGISTRY and the real backend's real applicable_guards set.
A non-empty result means this backend would be bricked at dispatch time.
"""
backend = backend_cls()
blockers = _get_fix_required_hook_matchers(backend.capabilities.applicable_guards)
assert not blockers, (
f"Backend {backend_name!r} would be bricked at dispatch by fix-required "
f"hooks with matchers {blockers}. Either reclassify the hook's codex_status "
f"or add the missing guard to applicable_guards."
)


@pytest.mark.parametrize("backend_name,backend_cls", sorted(BACKEND_REGISTRY.items()))
def test_applicable_guards_covers_all_fix_required_scripts(
backend_name: str, backend_cls: type
) -> None:
"""Set-level invariant: applicable_guards ⊇ fix-required hook script stems.

A parallel, structural assertion that doesn't go through the gate function.
Detects the same class of bug from the registry-composition angle.
"""
backend = backend_cls()
fix_required_stems = _collect_fix_required_stems()
missing = fix_required_stems - backend.capabilities.applicable_guards
assert not missing, (
f"Backend {backend_name!r} applicable_guards is missing script stems "
f"{sorted(missing)} required by fix-required hooks in HOOK_REGISTRY."
)


def test_test_matrix_covers_all_registered_backends() -> None:
"""Meta-test: the parametrized matrix must cover every backend in BACKEND_REGISTRY.

Prevents silent matrix shrinkage if a backend is removed from parametrization.
"""
expected = {"claude-code", "codex"}
actual = set(BACKEND_REGISTRY.keys())
assert len(actual) >= 2, (
f"BACKEND_REGISTRY has only {len(actual)} backend(s) — "
f"expected at least 2 (claude-code + codex). If a backend was removed, "
f"update this lower bound."
)
assert actual == expected, (
f"BACKEND_REGISTRY keys {sorted(actual)} != expected {sorted(expected)}. "
f"Update the expected set in this meta-test when backends are added or removed."
)
2 changes: 1 addition & 1 deletion tests/infra/test_schema_version_convention.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def _is_yaml_dump(node: ast.expr) -> bool:
# staleness_cache.py — cache dict
("src/autoskillit/recipe/staleness_cache.py", 67),
# _lifespan.py — hooks.json self-heal on startup drift (co-owned with Claude plugin system)
("src/autoskillit/server/_lifespan.py", 87),
("src/autoskillit/server/_lifespan.py", 89),
# tools_kitchen.py — hook config, quota guard, git_ops_policy, ingredient locks overlay
("src/autoskillit/server/tools/tools_kitchen.py", 167),
("src/autoskillit/server/tools/tools_kitchen.py", 186),
Expand Down
Loading