Skip to content

Commit 968d1e8

Browse files
Trecekclaude
andcommitted
test: add capability admission control tests and structural guards
- test_codex_implementation_dispatch_infeasible: codex + implementation recipe reports dispatch_feasible=False with gate_backend_write infeasible - test_claude_implementation_dispatch_feasible: claude-code backend reports dispatch_feasible=True - test_dispatch_feasible_defaults_true: recipes without capability gates default to feasible - E2E chain test verifies backend → load_and_validate → dispatch_feasible - AST-level tests guard all four content-serving surfaces and the load_and_validate call site - CAPABILITY_GATE_CALLABLES ↔ smoke_utils export sync test Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent e30abb5 commit 968d1e8

5 files changed

Lines changed: 213 additions & 0 deletions

File tree

tests/arch/AGENTS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ AST enforcement, sub-package layer contracts, and architectural invariant tests.
1717
| `test_backend_flag_isolation.py` | AST guard: ClaudeFlags must not appear in _session_launch.py — backend-specific flags belong inside each backend's build_interactive_cmd() |
1818
| `test_backend_coherence.py` | Architectural tests for backend coherence enforcement |
1919
| `test_canonical_constant_consumption.py` | Architectural invariant: every *_ENV_FORWARD_VARS constant must have a production consumer |
20+
| `test_capability_admission_control.py` | Architectural structural tests for capability admission control: dispatch_feasible gating on all four content-serving surfaces, _compute_capability_feasibility call in load_and_validate, CAPABILITY_GATE_CALLABLES/BACKEND_CAPABILITY_INGREDIENTS registry sync |
2021
| `test_backend_name_sync.py` | Architectural invariant: KNOWN_BACKEND_NAMES (IL-0) must match BACKEND_REGISTRY keys (IL-1) |
2122
| `test_capability_consistency.py` | Behavioral arch tests: BackendCapabilities filesystem consistency — applicable guards exist on disk, required session files are created, session_dir_symlinks entries are symlinks |
2223
| `test_cross_registry_dispatch_sufficiency.py` | Cross-registry dispatch sufficiency: real HOOK_REGISTRY × real BackendCapabilities — detects fix-required hooks that would brick backends with insufficient applicable_guards |
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
"""Architectural structural tests for capability admission control.
2+
3+
Verifies that:
4+
- _compute_capability_feasibility is called inside load_and_validate
5+
- All four content-serving surfaces gate on dispatch_feasible
6+
- CAPABILITY_GATE_CALLABLES entries have a corresponding BACKEND_CAPABILITY_INGREDIENTS input
7+
- Every run_python step using a CAPABILITY_GATE_CALLABLES callable reads a capability ingredient
8+
"""
9+
10+
from __future__ import annotations
11+
12+
import ast
13+
14+
import pytest
15+
16+
from tests.arch._helpers import SRC_ROOT
17+
18+
pytestmark = [pytest.mark.layer("arch"), pytest.mark.small]
19+
20+
21+
def _has_call_in_function(tree: ast.Module, func_name: str, target_call: str) -> bool:
22+
"""Return True if the named function contains a call to target_call."""
23+
for node in ast.walk(tree):
24+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
25+
if node.name != func_name:
26+
continue
27+
for child in ast.walk(node):
28+
if (
29+
isinstance(child, ast.Call)
30+
and isinstance(child.func, ast.Name)
31+
and child.func.id == target_call
32+
):
33+
return True
34+
return False
35+
36+
37+
def test_load_and_validate_calls_compute_capability_feasibility() -> None:
38+
"""load_and_validate must invoke _compute_capability_feasibility."""
39+
api_path = SRC_ROOT / "recipe" / "_api.py"
40+
tree = ast.parse(api_path.read_text(encoding="utf-8"))
41+
assert _has_call_in_function(tree, "load_and_validate", "_compute_capability_feasibility"), (
42+
"load_and_validate must call _compute_capability_feasibility "
43+
"to detect DOA pipelines from capability-gated run_python steps."
44+
)
45+
46+
47+
def test_open_kitchen_gates_on_dispatch_feasible() -> None:
48+
"""open_kitchen must check dispatch_feasible before enabling the gate."""
49+
kitchen_path = SRC_ROOT / "server" / "tools" / "tools_kitchen.py"
50+
src = kitchen_path.read_text(encoding="utf-8")
51+
tree = ast.parse(src)
52+
for node in ast.walk(tree):
53+
if isinstance(node, ast.AsyncFunctionDef) and node.name == "open_kitchen":
54+
src_dispatch_feasible = False
55+
for child in ast.walk(node):
56+
if isinstance(child, ast.Constant) and child.value == "dispatch_feasible":
57+
src_dispatch_feasible = True
58+
break
59+
assert src_dispatch_feasible, (
60+
"open_kitchen must reference 'dispatch_feasible' to gate "
61+
"capability-DOA pipelines before revealing tools."
62+
)
63+
return
64+
pytest.fail("open_kitchen function not found in tools_kitchen.py")
65+
66+
67+
def test_get_recipe_gates_on_dispatch_feasible() -> None:
68+
"""get_recipe resource must check dispatch_feasible."""
69+
kitchen_path = SRC_ROOT / "server" / "tools" / "tools_kitchen.py"
70+
src = kitchen_path.read_text(encoding="utf-8")
71+
assert "dispatch_feasible" in src, (
72+
"get_recipe resource must reference 'dispatch_feasible' to gate "
73+
"capability-DOA recipe serving."
74+
)
75+
76+
77+
def test_load_recipe_gates_on_dispatch_feasible() -> None:
78+
"""load_recipe tool must check dispatch_feasible."""
79+
recipe_path = SRC_ROOT / "server" / "tools" / "tools_recipe.py"
80+
src = recipe_path.read_text(encoding="utf-8")
81+
assert "dispatch_feasible" in src, (
82+
"load_recipe tool must reference 'dispatch_feasible' to surface "
83+
"capability-DOA signal to calling orchestrators."
84+
)
85+
86+
87+
def test_capability_gate_callables_have_matching_ingredient() -> None:
88+
"""Every CAPABILITY_GATE_CALLABLES callable must map to a BACKEND_CAPABILITY_INGREDIENTS input.
89+
90+
This guards against adding a new gate callable without declaring its
91+
capability ingredient dependency.
92+
"""
93+
import autoskillit.smoke_utils as smoke_utils_mod
94+
from autoskillit.core import BACKEND_CAPABILITY_INGREDIENTS, CAPABILITY_GATE_CALLABLES
95+
96+
for callable_name in CAPABILITY_GATE_CALLABLES:
97+
assert hasattr(smoke_utils_mod, callable_name), (
98+
f"CAPABILITY_GATE_CALLABLES entry {callable_name!r} has no matching "
99+
f"callable in autoskillit.smoke_utils."
100+
)
101+
assert callable_name in smoke_utils_mod.__all__, (
102+
f"CAPABILITY_GATE_CALLABLES entry {callable_name!r} is not exported "
103+
f"via smoke_utils.__all__."
104+
)
105+
106+
assert len(BACKEND_CAPABILITY_INGREDIENTS) > 0, (
107+
"BACKEND_CAPABILITY_INGREDIENTS must declare at least one capability ingredient key."
108+
)
109+
assert len(CAPABILITY_GATE_CALLABLES) > 0, (
110+
"CAPABILITY_GATE_CALLABLES must declare at least one capability gate callable."
111+
)

tests/recipe/test_bundled_recipes_dispatch_ready.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,3 +258,48 @@ def test_card_and_semantic_rules_agree_on_errors(recipe_name: str, tmp_path: Pat
258258
f"Card-only: {contract_error_steps - semantic_error_steps}, "
259259
f"Semantic-only: {semantic_error_steps - contract_error_steps}"
260260
)
261+
262+
263+
# ---------------------------------------------------------------------------
264+
# Capability-admission-control feasibility signal
265+
# ---------------------------------------------------------------------------
266+
267+
268+
def test_codex_implementation_dispatch_infeasible() -> None:
269+
"""Codex backend + implementation recipe must report dispatch_feasible=False.
270+
271+
gate_backend_write routes to terminal failure when backend_capable=false.
272+
"""
273+
result = load_and_validate(
274+
"implementation",
275+
project_dir=_PROJECT_ROOT,
276+
ingredient_overrides={"backend_supports_git_write": "false"},
277+
backend_name="codex",
278+
)
279+
assert result["valid"] is True # recipe is structurally valid
280+
assert result.get("dispatch_feasible") is False # but pipeline is DOA
281+
assert "gate_backend_write" in result.get("infeasible_steps", [])
282+
283+
284+
def test_claude_implementation_dispatch_feasible() -> None:
285+
"""Claude Code backend + implementation recipe must report dispatch_feasible=True."""
286+
result = load_and_validate(
287+
"implementation",
288+
project_dir=_PROJECT_ROOT,
289+
ingredient_overrides={"backend_supports_git_write": "true"},
290+
backend_name="claude-code",
291+
)
292+
assert result["valid"] is True
293+
assert result.get("dispatch_feasible") is True
294+
295+
296+
def test_dispatch_feasible_defaults_true_for_recipes_without_capability_gates() -> None:
297+
"""Recipes that do not use capability-gated run_python steps should
298+
default to dispatch_feasible=True regardless of backend overrides."""
299+
result = load_and_validate(
300+
"implementation",
301+
project_dir=_PROJECT_ROOT,
302+
ingredient_overrides={"backend_supports_git_write": "true"},
303+
)
304+
assert result.get("dispatch_feasible") is True
305+
assert "infeasible_steps" not in result

tests/server/AGENTS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Server tool handler unit tests — kitchen, execution, CI, clone, workspace tool
1010
| `_helpers.py` | Shared test builder utilities for tests/server/ |
1111
| `_type_coercion_fixtures.py` | Test fixtures for _import_and_call annotation-aware type coercion |
1212
| `conftest.py` | Shared fixtures for tests/server/ |
13+
| `test_capability_admission_e2e.py` | End-to-end chain tests for capability admission control: backend → load_and_validate → dispatch_feasible signal |
1314
| `test_editable_guard.py` | Unit tests for server/_editable_guard.py — scan_editable_installs_for_worktree |
1415
| `test_factory.py` | Tests for server/_factory.py make_context() composition root |
1516
| `test_factory_recording.py` | Tests for make_context recording/replay runner wiring |
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""End-to-end chain tests for capability admission control.
2+
3+
Verifies the full chain from backend capability detection through
4+
load_and_validate to the dispatch_feasible signal.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
import pytest
10+
11+
from autoskillit.core import BACKEND_CAPABILITY_INGREDIENTS, CAPABILITY_GATE_CALLABLES
12+
from autoskillit.recipe._api import load_and_validate
13+
14+
pytestmark = [pytest.mark.layer("server"), pytest.mark.medium]
15+
16+
17+
_PROJECT_ROOT = pytest.importorskip("pathlib").Path(__file__).resolve().parent.parent.parent
18+
19+
20+
def test_capability_ingredient_keys_match_registry() -> None:
21+
"""BACKEND_CAPABILITY_INGREDIENTS must include backend_supports_git_write."""
22+
assert "backend_supports_git_write" in BACKEND_CAPABILITY_INGREDIENTS
23+
24+
25+
def test_capability_gate_callables_includes_gate_backend_write() -> None:
26+
"""CAPABILITY_GATE_CALLABLES must include gate_backend_write."""
27+
assert "gate_backend_write" in CAPABILITY_GATE_CALLABLES
28+
29+
30+
def test_codex_backend_produces_infeasible_recipe() -> None:
31+
"""Codex + implementation recipe chain: backend_supports_git_write=false
32+
produces dispatch_feasible=False with gate_backend_write infeasible."""
33+
result = load_and_validate(
34+
"implementation",
35+
project_dir=_PROJECT_ROOT,
36+
ingredient_overrides={"backend_supports_git_write": "false"},
37+
backend_name="codex",
38+
)
39+
assert result.get("valid") is True
40+
assert result.get("dispatch_feasible") is False
41+
assert "gate_backend_write" in result.get("infeasible_steps", [])
42+
43+
44+
def test_claude_code_backend_produces_feasible_recipe() -> None:
45+
"""Claude Code + implementation recipe: backend_supports_git_write=true
46+
produces dispatch_feasible=True."""
47+
result = load_and_validate(
48+
"implementation",
49+
project_dir=_PROJECT_ROOT,
50+
ingredient_overrides={"backend_supports_git_write": "true"},
51+
backend_name="claude-code",
52+
)
53+
assert result.get("valid") is True
54+
assert result.get("dispatch_feasible") is True
55+
assert "infeasible_steps" not in result

0 commit comments

Comments
 (0)