Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion src/autoskillit/recipe/_recipe_composition.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ def _compute_capability_feasibility(
skip_resolutions=skip_resolutions,
pre_prune_steps=pre_prune_steps,
post_prune_steps=steps,
post_prune_recipe=post_prune_recipe,
):
continue
infeasible.append(step_name)
Expand All @@ -182,8 +183,15 @@ def _is_vacuous_gate(
skip_resolutions: dict[str, bool | None] | None,
pre_prune_steps: dict[str, Any] | None,
post_prune_steps: dict[str, Any],
post_prune_recipe: Any,
) -> bool:
"""Return True if the gate is vacuous — its guarded operations were all pruned."""
"""Return True if the gate is vacuous — its guarded operations were all pruned
AND the gate step is unreachable in the post-prune flow graph.

A gate whose guarded steps are all pruned is only vacuous if no surviving
step routes to it. Route-repair in `_prune_skipped_steps` can redirect
upstream steps directly to the gate, making it reachable and executable.
"""
if skip_resolutions is None or pre_prune_steps is None:
return False
for cap_key in gate_input_keys:
Expand All @@ -210,9 +218,34 @@ def _is_vacuous_gate(
break
if live_uses_capability:
return False
if _gate_reachable_post_prune(post_prune_recipe, gate_step_name):
return False
return True


def _gate_reachable_post_prune(post_prune_recipe: Any, gate_step_name: str) -> bool:
"""Return True if any surviving step routes to gate_step_name.

Uses `_collect_all_route_targets` to check all routing fields
(on_success, on_failure, on_context_limit, on_rate_limit, on_exhausted,
on_result conditions/routes). A gate with at least one incoming routing
edge from a surviving step is considered reachable.

BFS from entry is insufficient because unrelated pruning (e.g.
claim_and_resolve with a default-condition redirect to a failure step)
can disconnect the entry step from the main pipeline while the gate
remains routable via other surviving steps.
"""
if not post_prune_recipe or not getattr(post_prune_recipe, "steps", None):
return False
for step_name, step in post_prune_recipe.steps.items():
if step_name == gate_step_name:
continue
if gate_step_name in _collect_all_route_targets(step):
return True
return False


def _drop_sub_recipe_step(recipe: Any, step_name: str) -> Any:
"""Return a new Recipe with the named sub_recipe placeholder step removed."""
new_steps = {k: v for k, v in recipe.steps.items() if k != step_name}
Expand Down
8 changes: 8 additions & 0 deletions src/autoskillit/server/tools/tools_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,14 @@ async def load_recipe(
f"Recipe is infeasible on current backend: "
f"steps {result.get('infeasible_steps', [])} route to terminal failure."
)
return json.dumps(
{
"success": False,
"dispatch_infeasible": True,
"infeasible_steps": result.get("infeasible_steps", []),
"user_visible_message": result["user_visible_message"],
}
)
if ingredients_only:
result = strip_ingredients_only_keys(result)
_required_keys: frozenset[str] = frozenset()
Expand Down
28 changes: 28 additions & 0 deletions tests/arch/test_capability_admission_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,34 @@ def test_compute_capability_feasibility_receives_skip_resolutions() -> None:
pytest.fail("load_and_validate function not found in _api.py")


def test_compute_capability_feasibility_forwards_post_prune_recipe() -> None:
"""_compute_capability_feasibility must pass post_prune_recipe to
_is_vacuous_gate so the reachability guard has access to the full Recipe
object for graph analysis."""
comp_path = SRC_ROOT / "recipe" / "_recipe_composition.py"
tree = ast.parse(comp_path.read_text(encoding="utf-8"))
for node in ast.walk(tree):
if (
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name == "_compute_capability_feasibility"
):
for child in ast.walk(node):
if (
isinstance(child, ast.Call)
and isinstance(child.func, ast.Name)
and child.func.id == "_is_vacuous_gate"
):
kw_names = {kw.arg for kw in child.keywords if kw.arg is not None}
assert "post_prune_recipe" in kw_names, (
"_compute_capability_feasibility must pass post_prune_recipe "
"kwarg to _is_vacuous_gate for reachability-aware vacuity "
"detection."
)
return
pytest.fail("_is_vacuous_gate call not found in _compute_capability_feasibility")
pytest.fail("_compute_capability_feasibility function not found in _recipe_composition.py")


def test_dispatch_food_truck_injects_capability_overrides() -> None:
"""dispatch_food_truck must reference _build_capability_overrides to inject
backend capability signals into the load_and_validate call."""
Expand Down
1 change: 1 addition & 0 deletions tests/recipe/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ Recipe I/O, validation, semantic rules, schema, and bundled recipe tests.
| `test_promote_to_main_wrapper.py` | Tests for the promote-to-main wrapper recipe |
| `test_pseudocode_sync_rule.py` | Tests for the pseudocode-callable-divergence semantic rule |
| `test_recipe_backend_composition_matrix.py` | Recipe x backend composition matrix: parametrized CI gate validating every (recipe, backend) combination with declared-unsupported and known-broken governance |
| `test_recipe_composition_vacuous_gate.py` | Tests for reachability-aware `_is_vacuous_gate`: gate reachable post-prune is NOT vacuous, unreachable gate IS vacuous, all bundled gate-equipped recipes must report dispatch_feasible=False under codex |
| `test_recipe_ci_applicable_routing.py` | Structural tests for ci_applicable routing guards across all wait_for_ci chains |
| `test_recipe_ci_contracts.py` | Cross-recipe ci_event/branch coherence and remote_url structural tests |
| `test_recipe_ci_watch_event.py` | Tests for CI watch event in recipe steps |
Expand Down
43 changes: 24 additions & 19 deletions tests/recipe/test_bundled_recipes_dispatch_ready.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,20 +266,22 @@ def test_card_and_semantic_rules_agree_on_errors(recipe_name: str, tmp_path: Pat


def test_codex_implementation_dispatch_infeasible() -> None:
"""Codex backend + implementation recipe must report dispatch_feasible=True
when gate_backend_write is vacuous (all guarded steps were pruned)."""
"""Codex backend + implementation recipe must report dispatch_feasible=False
when gate_backend_write is reachable post-prune (route-repair redirects
upstream steps to the gate after implement is pruned)."""
result = load_and_validate(
"implementation",
project_dir=_PROJECT_ROOT,
ingredient_overrides={"backend_supports_git_write": "false"},
backend_name="codex",
)
assert result["valid"] is True
assert result.get("dispatch_feasible") is True, (
"When all backend_supports_git_write-gated steps are pruned, "
"gate_backend_write is vacuous and should NOT block dispatch."
assert result.get("dispatch_feasible") is False, (
"When implement is pruned but create_impl_worktree.on_success is "
"route-repaired to gate_backend_write, the gate is reachable and "
"must block dispatch as infeasible."
)
assert "gate_backend_write" not in result.get("infeasible_steps", [])
assert "gate_backend_write" in result.get("infeasible_steps", [])


def test_claude_implementation_dispatch_feasible() -> None:
Expand Down Expand Up @@ -328,23 +330,26 @@ def _discover_capability_gate_recipes() -> list[str]:

@pytest.mark.parametrize("recipe_name", _CAPABILITY_GATE_RECIPES)
def test_codex_capability_gate_recipes(recipe_name: str) -> None:
"""Every recipe with a gate_backend_write step must pass vacuous-gate
detection under codex overrides — either dispatch_feasible=True (gate
is vacuous) or dispatch_feasible=False with gate in infeasible_steps
(gate guards live steps)."""
"""Every recipe with a gate_backend_write step must report dispatch_feasible=False
with gate_backend_write in infeasible_steps when the gate is reachable post-prune.

Route-repair in _prune_skipped_steps redirects create_impl_worktree.on_success
to gate_backend_write after pruning the guarded steps, making the gate reachable
from the entry step. Admission control must identify this as an infeasible pipeline.
"""
result = load_and_validate(
recipe_name,
project_dir=_PROJECT_ROOT,
ingredient_overrides={"backend_supports_git_write": "false"},
backend_name="codex",
)
assert result.get("valid") is True
dispatch_feasible = result.get("dispatch_feasible")
infeasible_steps = result.get("infeasible_steps", [])
if dispatch_feasible is True:
assert "gate_backend_write" not in infeasible_steps
else:
assert dispatch_feasible is False, (
f"Expected dispatch_feasible to be explicitly False, got {dispatch_feasible!r}"
)
assert "gate_backend_write" in infeasible_steps
assert result.get("dispatch_feasible") is False, (
f"Recipe '{recipe_name}' must report dispatch_feasible=False when "
f"gate_backend_write is reachable post-prune; got: "
f"{result.get('dispatch_feasible')!r}"
)
assert "gate_backend_write" in result.get("infeasible_steps", []), (
f"Recipe '{recipe_name}' must list gate_backend_write in infeasible_steps; "
f"got: {result.get('infeasible_steps')}"
)
185 changes: 185 additions & 0 deletions tests/recipe/test_recipe_composition_vacuous_gate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
"""Tests for reachability-aware vacuous gate detection in _is_vacuous_gate.

The vacuous gate exemption (`_is_vacuous_gate` returning True) allowed
dispatch to pass admission control when all guarded steps were pruned —
but route-repair in `_prune_skipped_steps` can redirect upstream steps
directly to the gate, making it reachable and executable in the
post-prune flow graph. These tests pin the correct behavior:

- A gate reachable from the entry step is NOT vacuous (it will execute
and fail at runtime → must be reported as infeasible).
- A gate with no incoming routing edges is vacuous (all guards were
pruned and nothing routes to it → safe to exempt).
"""

from __future__ import annotations

from dataclasses import dataclass, field
from pathlib import Path

import pytest

pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small]


_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent


@dataclass
class _StubStep:
tool: str | None = None
action: str | None = None
note: str | None = None
with_args: dict | None = None
on_success: str | None = None
on_failure: str | None = None
on_context_limit: str | None = None
on_rate_limit: str | None = None
on_exhausted: str | None = None
on_result: object | None = None
skip_when_false: str | None = None
optional: bool = False
sub_recipe: str | None = None


@dataclass
class _StubRecipe:
steps: dict = field(default_factory=dict)
ingredients: dict = field(default_factory=dict)


class _StubPreStep:
def __init__(self, skip_when_false: str) -> None:
self.skip_when_false = skip_when_false


def _build_recipe(steps: dict[str, dict]) -> _StubRecipe:
"""Build a Recipe-like object from a flat step-dict (name -> fields)."""
stub_steps: dict[str, _StubStep] = {}
for step_name, fields in steps.items():
stub_steps[step_name] = _StubStep(**fields)
return _StubRecipe(steps=stub_steps)


def test_is_vacuous_gate_returns_false_when_gate_reachable_post_prune() -> None:
"""Gate is reachable via create_impl_worktree.on_success → gate_backend_write.

With `implement` pruned but `create_impl_worktree.on_success` redirected to
`gate_backend_write` by route-repair, the gate is reachable from the entry
step. _is_vacuous_gate must return False (not vacuous) → infeasible.
"""
from autoskillit.recipe._recipe_composition import _is_vacuous_gate

recipe = _build_recipe(
{
"create_impl_worktree": {"on_success": "gate_backend_write"},
"gate_backend_write": {
"tool": "run_python",
"with_args": {
"callable": "autoskillit.smoke_utils.gate_backend_write",
"backend_supports_git_write": "false",
},
"on_failure": "escalate",
"on_exhausted": "escalate",
},
}
)

gate_input_keys = {"backend_supports_git_write"}
pre_prune_steps = {
"implement": _StubPreStep(skip_when_false="inputs.backend_supports_git_write"),
}
skip_resolutions: dict = {"implement": False}
post_prune_steps = recipe.steps

result = _is_vacuous_gate(
gate_input_keys,
gate_step_name="gate_backend_write",
skip_resolutions=skip_resolutions,
pre_prune_steps=pre_prune_steps,
post_prune_steps=post_prune_steps,
post_prune_recipe=recipe,
)

assert result is False, (
"Gate is reachable via create_impl_worktree.on_success after route-repair; "
"must NOT be treated as vacuous."
)


def test_is_vacuous_gate_returns_true_when_gate_unreachable_post_prune() -> None:
"""Gate has no incoming routing edges (truly unreachable after pruning).

When all guarded steps are pruned AND no other step routes to the gate,
the gate is truly unreachable. _is_vacuous_gate must return True (vacuous).
"""
from autoskillit.recipe._recipe_composition import _is_vacuous_gate

recipe = _build_recipe(
{
"create_impl_worktree": {"on_success": "done"},
"gate_backend_write": {
"tool": "run_python",
"with_args": {
"callable": "autoskillit.smoke_utils.gate_backend_write",
"backend_supports_git_write": "false",
},
"on_failure": "escalate",
"on_exhausted": "escalate",
},
}
)

gate_input_keys = {"backend_supports_git_write"}
pre_prune_steps = {
"implement": _StubPreStep(skip_when_false="inputs.backend_supports_git_write"),
}
skip_resolutions: dict = {"implement": False}
post_prune_steps = recipe.steps

result = _is_vacuous_gate(
gate_input_keys,
gate_step_name="gate_backend_write",
skip_resolutions=skip_resolutions,
pre_prune_steps=pre_prune_steps,
post_prune_steps=post_prune_steps,
post_prune_recipe=recipe,
)

assert result is True, (
"Gate has no incoming routing edges and all guards pruned; must be treated as vacuous."
)


@pytest.mark.parametrize(
"recipe_name",
["implementation", "remediation", "implementation-groups"],
)
def test_compute_capability_feasibility_returns_infeasible_for_codex_recipes(
recipe_name: str,
) -> None:
"""All three gate-equipped recipes must report dispatch_feasible=False
when backend_supports_git_write=false under codex backend.

Route-repair redirects create_impl_worktree.on_success to gate_backend_write
after pruning `implement`, making the gate reachable. Admission control
must identify this as an infeasible pipeline.
"""
from autoskillit.recipe._api import load_and_validate

result = load_and_validate(
recipe_name,
project_dir=_PROJECT_ROOT,
ingredient_overrides={"backend_supports_git_write": "false"},
backend_name="codex",
)
assert result["valid"] is True
assert result.get("dispatch_feasible") is False, (
f"Recipe '{recipe_name}' with backend_supports_git_write=false under codex "
f"must report dispatch_feasible=False (gate is reachable post-prune); "
f"got: {result.get('dispatch_feasible')}"
)
assert "gate_backend_write" in result.get("infeasible_steps", []), (
f"Recipe '{recipe_name}' must list gate_backend_write in infeasible_steps; "
f"got: {result.get('infeasible_steps')}"
)
Loading
Loading