diff --git a/src/autoskillit/recipe/_recipe_composition.py b/src/autoskillit/recipe/_recipe_composition.py index fc791d8a0..0a085cea0 100644 --- a/src/autoskillit/recipe/_recipe_composition.py +++ b/src/autoskillit/recipe/_recipe_composition.py @@ -168,6 +168,7 @@ def _compute_capability_feasibility( skip_resolutions=skip_resolutions, pre_prune_steps=pre_prune_steps, post_prune_steps=steps, + post_prune_recipe=post_prune_recipe, ): continue infeasible.append(step_name) @@ -182,8 +183,15 @@ def _is_vacuous_gate( skip_resolutions: dict[str, bool | None] | None, pre_prune_steps: dict[str, Any] | None, post_prune_steps: dict[str, Any], + post_prune_recipe: Any, ) -> bool: - """Return True if the gate is vacuous — its guarded operations were all pruned.""" + """Return True if the gate is vacuous — its guarded operations were all pruned + AND the gate step is unreachable in the post-prune flow graph. + + A gate whose guarded steps are all pruned is only vacuous if no surviving + step routes to it. Route-repair in `_prune_skipped_steps` can redirect + upstream steps directly to the gate, making it reachable and executable. + """ if skip_resolutions is None or pre_prune_steps is None: return False for cap_key in gate_input_keys: @@ -210,9 +218,34 @@ def _is_vacuous_gate( break if live_uses_capability: return False + if _gate_reachable_post_prune(post_prune_recipe, gate_step_name): + return False return True +def _gate_reachable_post_prune(post_prune_recipe: Any, gate_step_name: str) -> bool: + """Return True if any surviving step routes to gate_step_name. + + Uses `_collect_all_route_targets` to check all routing fields + (on_success, on_failure, on_context_limit, on_rate_limit, on_exhausted, + on_result conditions/routes). A gate with at least one incoming routing + edge from a surviving step is considered reachable. + + BFS from entry is insufficient because unrelated pruning (e.g. + claim_and_resolve with a default-condition redirect to a failure step) + can disconnect the entry step from the main pipeline while the gate + remains routable via other surviving steps. + """ + if not post_prune_recipe or not getattr(post_prune_recipe, "steps", None): + return False + for step_name, step in post_prune_recipe.steps.items(): + if step_name == gate_step_name: + continue + if gate_step_name in _collect_all_route_targets(step): + return True + return False + + def _drop_sub_recipe_step(recipe: Any, step_name: str) -> Any: """Return a new Recipe with the named sub_recipe placeholder step removed.""" new_steps = {k: v for k, v in recipe.steps.items() if k != step_name} diff --git a/src/autoskillit/server/tools/tools_recipe.py b/src/autoskillit/server/tools/tools_recipe.py index 026f39ceb..0767cb7bb 100644 --- a/src/autoskillit/server/tools/tools_recipe.py +++ b/src/autoskillit/server/tools/tools_recipe.py @@ -243,6 +243,14 @@ async def load_recipe( f"Recipe is infeasible on current backend: " f"steps {result.get('infeasible_steps', [])} route to terminal failure." ) + return json.dumps( + { + "success": False, + "dispatch_infeasible": True, + "infeasible_steps": result.get("infeasible_steps", []), + "user_visible_message": result["user_visible_message"], + } + ) if ingredients_only: result = strip_ingredients_only_keys(result) _required_keys: frozenset[str] = frozenset() diff --git a/tests/arch/test_capability_admission_control.py b/tests/arch/test_capability_admission_control.py index 20a3a9a84..08f79b6d7 100644 --- a/tests/arch/test_capability_admission_control.py +++ b/tests/arch/test_capability_admission_control.py @@ -152,6 +152,34 @@ def test_compute_capability_feasibility_receives_skip_resolutions() -> None: pytest.fail("load_and_validate function not found in _api.py") +def test_compute_capability_feasibility_forwards_post_prune_recipe() -> None: + """_compute_capability_feasibility must pass post_prune_recipe to + _is_vacuous_gate so the reachability guard has access to the full Recipe + object for graph analysis.""" + comp_path = SRC_ROOT / "recipe" / "_recipe_composition.py" + tree = ast.parse(comp_path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == "_compute_capability_feasibility" + ): + for child in ast.walk(node): + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Name) + and child.func.id == "_is_vacuous_gate" + ): + kw_names = {kw.arg for kw in child.keywords if kw.arg is not None} + assert "post_prune_recipe" in kw_names, ( + "_compute_capability_feasibility must pass post_prune_recipe " + "kwarg to _is_vacuous_gate for reachability-aware vacuity " + "detection." + ) + return + pytest.fail("_is_vacuous_gate call not found in _compute_capability_feasibility") + pytest.fail("_compute_capability_feasibility function not found in _recipe_composition.py") + + def test_dispatch_food_truck_injects_capability_overrides() -> None: """dispatch_food_truck must reference _build_capability_overrides to inject backend capability signals into the load_and_validate call.""" diff --git a/tests/recipe/AGENTS.md b/tests/recipe/AGENTS.md index 5b837b714..529c3c897 100644 --- a/tests/recipe/AGENTS.md +++ b/tests/recipe/AGENTS.md @@ -84,6 +84,7 @@ Recipe I/O, validation, semantic rules, schema, and bundled recipe tests. | `test_promote_to_main_wrapper.py` | Tests for the promote-to-main wrapper recipe | | `test_pseudocode_sync_rule.py` | Tests for the pseudocode-callable-divergence semantic rule | | `test_recipe_backend_composition_matrix.py` | Recipe x backend composition matrix: parametrized CI gate validating every (recipe, backend) combination with declared-unsupported and known-broken governance | +| `test_recipe_composition_vacuous_gate.py` | Tests for reachability-aware `_is_vacuous_gate`: gate reachable post-prune is NOT vacuous, unreachable gate IS vacuous, all bundled gate-equipped recipes must report dispatch_feasible=False under codex | | `test_recipe_ci_applicable_routing.py` | Structural tests for ci_applicable routing guards across all wait_for_ci chains | | `test_recipe_ci_contracts.py` | Cross-recipe ci_event/branch coherence and remote_url structural tests | | `test_recipe_ci_watch_event.py` | Tests for CI watch event in recipe steps | diff --git a/tests/recipe/test_bundled_recipes_dispatch_ready.py b/tests/recipe/test_bundled_recipes_dispatch_ready.py index 01140a797..31729b294 100644 --- a/tests/recipe/test_bundled_recipes_dispatch_ready.py +++ b/tests/recipe/test_bundled_recipes_dispatch_ready.py @@ -266,8 +266,9 @@ def test_card_and_semantic_rules_agree_on_errors(recipe_name: str, tmp_path: Pat def test_codex_implementation_dispatch_infeasible() -> None: - """Codex backend + implementation recipe must report dispatch_feasible=True - when gate_backend_write is vacuous (all guarded steps were pruned).""" + """Codex backend + implementation recipe must report dispatch_feasible=False + when gate_backend_write is reachable post-prune (route-repair redirects + upstream steps to the gate after implement is pruned).""" result = load_and_validate( "implementation", project_dir=_PROJECT_ROOT, @@ -275,11 +276,12 @@ def test_codex_implementation_dispatch_infeasible() -> None: backend_name="codex", ) assert result["valid"] is True - assert result.get("dispatch_feasible") is True, ( - "When all backend_supports_git_write-gated steps are pruned, " - "gate_backend_write is vacuous and should NOT block dispatch." + assert result.get("dispatch_feasible") is False, ( + "When implement is pruned but create_impl_worktree.on_success is " + "route-repaired to gate_backend_write, the gate is reachable and " + "must block dispatch as infeasible." ) - assert "gate_backend_write" not in result.get("infeasible_steps", []) + assert "gate_backend_write" in result.get("infeasible_steps", []) def test_claude_implementation_dispatch_feasible() -> None: @@ -328,10 +330,13 @@ def _discover_capability_gate_recipes() -> list[str]: @pytest.mark.parametrize("recipe_name", _CAPABILITY_GATE_RECIPES) def test_codex_capability_gate_recipes(recipe_name: str) -> None: - """Every recipe with a gate_backend_write step must pass vacuous-gate - detection under codex overrides — either dispatch_feasible=True (gate - is vacuous) or dispatch_feasible=False with gate in infeasible_steps - (gate guards live steps).""" + """Every recipe with a gate_backend_write step must report dispatch_feasible=False + with gate_backend_write in infeasible_steps when the gate is reachable post-prune. + + Route-repair in _prune_skipped_steps redirects create_impl_worktree.on_success + to gate_backend_write after pruning the guarded steps, making the gate reachable + from the entry step. Admission control must identify this as an infeasible pipeline. + """ result = load_and_validate( recipe_name, project_dir=_PROJECT_ROOT, @@ -339,12 +344,12 @@ def test_codex_capability_gate_recipes(recipe_name: str) -> None: backend_name="codex", ) assert result.get("valid") is True - dispatch_feasible = result.get("dispatch_feasible") - infeasible_steps = result.get("infeasible_steps", []) - if dispatch_feasible is True: - assert "gate_backend_write" not in infeasible_steps - else: - assert dispatch_feasible is False, ( - f"Expected dispatch_feasible to be explicitly False, got {dispatch_feasible!r}" - ) - assert "gate_backend_write" in infeasible_steps + assert result.get("dispatch_feasible") is False, ( + f"Recipe '{recipe_name}' must report dispatch_feasible=False when " + f"gate_backend_write is reachable post-prune; got: " + f"{result.get('dispatch_feasible')!r}" + ) + assert "gate_backend_write" in result.get("infeasible_steps", []), ( + f"Recipe '{recipe_name}' must list gate_backend_write in infeasible_steps; " + f"got: {result.get('infeasible_steps')}" + ) diff --git a/tests/recipe/test_recipe_composition_vacuous_gate.py b/tests/recipe/test_recipe_composition_vacuous_gate.py new file mode 100644 index 000000000..133d27c3a --- /dev/null +++ b/tests/recipe/test_recipe_composition_vacuous_gate.py @@ -0,0 +1,185 @@ +"""Tests for reachability-aware vacuous gate detection in _is_vacuous_gate. + +The vacuous gate exemption (`_is_vacuous_gate` returning True) allowed +dispatch to pass admission control when all guarded steps were pruned — +but route-repair in `_prune_skipped_steps` can redirect upstream steps +directly to the gate, making it reachable and executable in the +post-prune flow graph. These tests pin the correct behavior: + +- A gate reachable from the entry step is NOT vacuous (it will execute + and fail at runtime → must be reported as infeasible). +- A gate with no incoming routing edges is vacuous (all guards were + pruned and nothing routes to it → safe to exempt). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent + + +@dataclass +class _StubStep: + tool: str | None = None + action: str | None = None + note: str | None = None + with_args: dict | None = None + on_success: str | None = None + on_failure: str | None = None + on_context_limit: str | None = None + on_rate_limit: str | None = None + on_exhausted: str | None = None + on_result: object | None = None + skip_when_false: str | None = None + optional: bool = False + sub_recipe: str | None = None + + +@dataclass +class _StubRecipe: + steps: dict = field(default_factory=dict) + ingredients: dict = field(default_factory=dict) + + +class _StubPreStep: + def __init__(self, skip_when_false: str) -> None: + self.skip_when_false = skip_when_false + + +def _build_recipe(steps: dict[str, dict]) -> _StubRecipe: + """Build a Recipe-like object from a flat step-dict (name -> fields).""" + stub_steps: dict[str, _StubStep] = {} + for step_name, fields in steps.items(): + stub_steps[step_name] = _StubStep(**fields) + return _StubRecipe(steps=stub_steps) + + +def test_is_vacuous_gate_returns_false_when_gate_reachable_post_prune() -> None: + """Gate is reachable via create_impl_worktree.on_success → gate_backend_write. + + With `implement` pruned but `create_impl_worktree.on_success` redirected to + `gate_backend_write` by route-repair, the gate is reachable from the entry + step. _is_vacuous_gate must return False (not vacuous) → infeasible. + """ + from autoskillit.recipe._recipe_composition import _is_vacuous_gate + + recipe = _build_recipe( + { + "create_impl_worktree": {"on_success": "gate_backend_write"}, + "gate_backend_write": { + "tool": "run_python", + "with_args": { + "callable": "autoskillit.smoke_utils.gate_backend_write", + "backend_supports_git_write": "false", + }, + "on_failure": "escalate", + "on_exhausted": "escalate", + }, + } + ) + + gate_input_keys = {"backend_supports_git_write"} + pre_prune_steps = { + "implement": _StubPreStep(skip_when_false="inputs.backend_supports_git_write"), + } + skip_resolutions: dict = {"implement": False} + post_prune_steps = recipe.steps + + result = _is_vacuous_gate( + gate_input_keys, + gate_step_name="gate_backend_write", + skip_resolutions=skip_resolutions, + pre_prune_steps=pre_prune_steps, + post_prune_steps=post_prune_steps, + post_prune_recipe=recipe, + ) + + assert result is False, ( + "Gate is reachable via create_impl_worktree.on_success after route-repair; " + "must NOT be treated as vacuous." + ) + + +def test_is_vacuous_gate_returns_true_when_gate_unreachable_post_prune() -> None: + """Gate has no incoming routing edges (truly unreachable after pruning). + + When all guarded steps are pruned AND no other step routes to the gate, + the gate is truly unreachable. _is_vacuous_gate must return True (vacuous). + """ + from autoskillit.recipe._recipe_composition import _is_vacuous_gate + + recipe = _build_recipe( + { + "create_impl_worktree": {"on_success": "done"}, + "gate_backend_write": { + "tool": "run_python", + "with_args": { + "callable": "autoskillit.smoke_utils.gate_backend_write", + "backend_supports_git_write": "false", + }, + "on_failure": "escalate", + "on_exhausted": "escalate", + }, + } + ) + + gate_input_keys = {"backend_supports_git_write"} + pre_prune_steps = { + "implement": _StubPreStep(skip_when_false="inputs.backend_supports_git_write"), + } + skip_resolutions: dict = {"implement": False} + post_prune_steps = recipe.steps + + result = _is_vacuous_gate( + gate_input_keys, + gate_step_name="gate_backend_write", + skip_resolutions=skip_resolutions, + pre_prune_steps=pre_prune_steps, + post_prune_steps=post_prune_steps, + post_prune_recipe=recipe, + ) + + assert result is True, ( + "Gate has no incoming routing edges and all guards pruned; must be treated as vacuous." + ) + + +@pytest.mark.parametrize( + "recipe_name", + ["implementation", "remediation", "implementation-groups"], +) +def test_compute_capability_feasibility_returns_infeasible_for_codex_recipes( + recipe_name: str, +) -> None: + """All three gate-equipped recipes must report dispatch_feasible=False + when backend_supports_git_write=false under codex backend. + + Route-repair redirects create_impl_worktree.on_success to gate_backend_write + after pruning `implement`, making the gate reachable. Admission control + must identify this as an infeasible pipeline. + """ + from autoskillit.recipe._api import load_and_validate + + result = load_and_validate( + recipe_name, + project_dir=_PROJECT_ROOT, + ingredient_overrides={"backend_supports_git_write": "false"}, + backend_name="codex", + ) + assert result["valid"] is True + assert result.get("dispatch_feasible") is False, ( + f"Recipe '{recipe_name}' with backend_supports_git_write=false under codex " + f"must report dispatch_feasible=False (gate is reachable post-prune); " + f"got: {result.get('dispatch_feasible')}" + ) + assert "gate_backend_write" in result.get("infeasible_steps", []), ( + f"Recipe '{recipe_name}' must list gate_backend_write in infeasible_steps; " + f"got: {result.get('infeasible_steps')}" + ) diff --git a/tests/server/test_backend_ingredient_injection.py b/tests/server/test_backend_ingredient_injection.py index d43c7601f..da5310478 100644 --- a/tests/server/test_backend_ingredient_injection.py +++ b/tests/server/test_backend_ingredient_injection.py @@ -496,11 +496,12 @@ def test_codex_overrides_remove_guarded_steps_from_content(self, tmp_path: Path) ) @pytest.mark.anyio - async def test_codex_open_kitchen_returns_success(self, tmp_path: Path) -> None: + async def test_codex_open_kitchen_blocks_on_reachable_gate(self, tmp_path: Path) -> None: """End-to-end integration: open_kitchen with codex backend on the bundled - implementation recipe must return success=True because vacuous-gate - detection recognises that all backend_supports_git_write-guarded steps - were pruned, making gate_backend_write vacuous. + implementation recipe must hard-block because gate_backend_write is reachable + post-prune (route-repair redirects create_impl_worktree.on_success to the + gate after implement is pruned). Admission control reports dispatch_feasible=False + and open_kitchen must reject the pipeline. """ from autoskillit.server.tools.tools_kitchen import open_kitchen @@ -554,11 +555,13 @@ def _real_load_wrapper(name, project_dir=None, **kwargs): result = json.loads(result_str) - assert result.get("success") is True, ( - f"open_kitchen must accept codex backend (vacuous gate); got: {result}" + assert result.get("success") is False, ( + f"open_kitchen must hard-block codex backend (reachable gate); got: {result}" ) - assert result.get("dispatch_feasible") is True, ( - "dispatch_feasible must be True when " - f"gate_backend_write is vacuous; got: " - f"{result.get('dispatch_feasible')}" + assert result.get("kitchen") == "dispatch_infeasible", ( + "kitchen must be 'dispatch_infeasible' when gate_backend_write is " + f"reachable post-prune; got: {result.get('kitchen')!r}" + ) + assert "gate_backend_write" in result.get("infeasible_steps", []), ( + f"infeasible_steps must list gate_backend_write; got: {result.get('infeasible_steps')}" ) diff --git a/tests/server/test_capability_admission_e2e.py b/tests/server/test_capability_admission_e2e.py index 51c26a123..d8d32d8f5 100644 --- a/tests/server/test_capability_admission_e2e.py +++ b/tests/server/test_capability_admission_e2e.py @@ -29,10 +29,11 @@ def test_capability_gate_callables_includes_gate_backend_write() -> None: assert "gate_backend_write" in CAPABILITY_GATE_CALLABLES -def test_codex_backend_produces_feasible_recipe_via_vacuous_gate() -> None: +def test_codex_backend_reachable_gate_returns_infeasible() -> None: """Codex + implementation recipe chain: backend_supports_git_write=false - produces dispatch_feasible=True because vacuous-gate detection recognises - all guarded steps were pruned.""" + produces dispatch_feasible=False because the reachable gate (post-prune + route-repair redirects create_impl_worktree.on_success to gate_backend_write) + blocks dispatch via admission control.""" result = load_and_validate( "implementation", project_dir=_PROJECT_ROOT, @@ -40,8 +41,8 @@ def test_codex_backend_produces_feasible_recipe_via_vacuous_gate() -> None: backend_name="codex", ) assert result.get("valid") is True - assert result.get("dispatch_feasible") is True - assert "gate_backend_write" not in result.get("infeasible_steps", []) + assert result.get("dispatch_feasible") is False + assert "gate_backend_write" in result.get("infeasible_steps", []) def test_claude_code_backend_produces_feasible_recipe() -> None: diff --git a/tests/server/test_tools_load_recipe.py b/tests/server/test_tools_load_recipe.py index 5322e8ad1..4711aace7 100644 --- a/tests/server/test_tools_load_recipe.py +++ b/tests/server/test_tools_load_recipe.py @@ -624,3 +624,33 @@ async def test_load_recipe_fail_closed_missing_content(self, monkeypatch, tmp_pa parsed = json.loads(raw) assert parsed["success"] is False assert "content" in parsed["error"] + + @pytest.mark.anyio + async def test_load_recipe_blocks_on_dispatch_infeasible(self, monkeypatch, tmp_path): + """load_recipe must hard-block when dispatch_feasible=False — no recipe + content is delivered to the caller when the pipeline is infeasible.""" + monkeypatch.chdir(tmp_path) + from autoskillit.recipe._api_cache import _LOAD_CACHE + + _LOAD_CACHE.clear() + test_result = { + "valid": True, + "content": "name: blocked-recipe\nsteps:\n build:\n cmd: task build\n", + "dispatch_feasible": False, + "infeasible_steps": ["gate_backend_write"], + } + monkeypatch.setattr(self.ctx.recipes, "load_and_validate", lambda *a, **kw: test_result) + monkeypatch.setattr(self.ctx.recipes, "find", lambda *a, **kw: None) + with patch( + "autoskillit.server.tools.tools_recipe._apply_triage_gate", + new=AsyncMock(return_value=test_result), + ): + raw = await load_recipe(name="blocked-recipe") + parsed = json.loads(raw) + assert parsed["success"] is False + assert parsed.get("dispatch_infeasible") is True + assert "gate_backend_write" in parsed.get("infeasible_steps", []) + assert "content" not in parsed, ( + "load_recipe must NOT deliver recipe content when dispatch is infeasible; " + f"got keys: {list(parsed.keys())}" + )