|
| 1 | +"""Recipe x backend composition matrix -- full cross-product CI gate. |
| 2 | +
|
| 3 | +Validates every bundled recipe composes validly under every registered backend. |
| 4 | +DECLARED_UNSUPPORTED governs by-design unsupported combos; orphan and |
| 5 | +collection-count meta-tests prevent skip-rot and matrix shrinkage. |
| 6 | +""" |
| 7 | + |
| 8 | +from __future__ import annotations |
| 9 | + |
| 10 | +from pathlib import Path |
| 11 | +from typing import Any |
| 12 | + |
| 13 | +import pytest |
| 14 | + |
| 15 | +from autoskillit.core import Severity |
| 16 | +from autoskillit.execution.backends import BACKEND_REGISTRY, get_backend |
| 17 | +from autoskillit.recipe._api import load_and_validate |
| 18 | +from autoskillit.recipe.io import all_validated_recipe_names |
| 19 | +from autoskillit.server.tools._auto_overrides import _backend_capability_overrides |
| 20 | +from autoskillit.workspace.skills import DefaultSkillResolver |
| 21 | + |
| 22 | +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.medium] |
| 23 | + |
| 24 | +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent |
| 25 | + |
| 26 | +_ALL_RECIPE_NAMES = sorted(all_validated_recipe_names(_PROJECT_ROOT)) |
| 27 | +_BACKEND_NAMES = sorted(BACKEND_REGISTRY.keys()) |
| 28 | + |
| 29 | + |
| 30 | +# -- By-design unsupported combos (skip) ------------------------------------ |
| 31 | +DECLARED_UNSUPPORTED: frozenset[tuple[str, str]] = frozenset() |
| 32 | + |
| 33 | +UNSUPPORTED_REASONS: dict[tuple[str, str], dict[str, str]] = {} |
| 34 | + |
| 35 | +_MATRIX_IDS: list[tuple[str, str]] = [ |
| 36 | + (r, b) for r in _ALL_RECIPE_NAMES for b in _BACKEND_NAMES if (r, b) not in DECLARED_UNSUPPORTED |
| 37 | +] |
| 38 | + |
| 39 | + |
| 40 | +# -- Known-broken combos (xfail strict) ------------------------------------- |
| 41 | +KNOWN_BROKEN: dict[tuple[str, str], str] = { |
| 42 | + ("agent-eval", "claude-code"): ( |
| 43 | + "tracking: #4069 -- violates all-dispatchable-stops-have-sentinel + dead-output" |
| 44 | + ), |
| 45 | + ("agent-eval", "codex"): ( |
| 46 | + "tracking: #4069 -- violates all-dispatchable-stops-have-sentinel + dead-output" |
| 47 | + ), |
| 48 | + ("skill-eval", "claude-code"): ( |
| 49 | + "tracking: #4069 -- violates all-dispatchable-stops-have-sentinel + dead-output" |
| 50 | + ), |
| 51 | + ("skill-eval", "codex"): ( |
| 52 | + "tracking: #4069 -- violates all-dispatchable-stops-have-sentinel + dead-output" |
| 53 | + ), |
| 54 | +} |
| 55 | + |
| 56 | +_SKILL_RESOLVER = DefaultSkillResolver() |
| 57 | + |
| 58 | + |
| 59 | +def _apply_marks(matrix_ids: list[tuple[str, str]]) -> list[Any]: |
| 60 | + """Wrap matrix tuples in pytest.param, attaching xfail marks for KNOWN_BROKEN.""" |
| 61 | + params: list[Any] = [] |
| 62 | + for r, b in matrix_ids: |
| 63 | + marks: list[Any] = [] |
| 64 | + if (r, b) in KNOWN_BROKEN: |
| 65 | + marks.append(pytest.mark.xfail(strict=True, reason=KNOWN_BROKEN[(r, b)])) |
| 66 | + params.append(pytest.param(r, b, marks=marks, id=f"{r}/{b}")) |
| 67 | + return params |
| 68 | + |
| 69 | + |
| 70 | +@pytest.mark.parametrize("recipe_name,backend_name", _apply_marks(_MATRIX_IDS)) |
| 71 | +def test_recipe_backend_matrix_cell(recipe_name: str, backend_name: str) -> None: |
| 72 | + backend = get_backend(backend_name) |
| 73 | + result = load_and_validate( |
| 74 | + recipe_name, |
| 75 | + project_dir=_PROJECT_ROOT, |
| 76 | + backend_name=backend_name, |
| 77 | + ingredient_overrides=_backend_capability_overrides(backend), |
| 78 | + lister=_SKILL_RESOLVER, |
| 79 | + ) |
| 80 | + |
| 81 | + assert result["valid"] is True, ( |
| 82 | + f"Recipe '{recipe_name}' invalid on backend '{backend_name}': " |
| 83 | + + "; ".join( |
| 84 | + f"[{s.get('rule')}] {s.get('message', '')[:80]}" |
| 85 | + for s in result.get("suggestions", []) |
| 86 | + if s.get("severity") == Severity.ERROR |
| 87 | + ) |
| 88 | + ) |
| 89 | + assert len(result.get("content", "")) > 0, ( |
| 90 | + f"Recipe '{recipe_name}' on backend '{backend_name}' produced empty content" |
| 91 | + ) |
| 92 | + |
| 93 | + suggestions: list[dict[str, Any]] = result.get("suggestions", []) |
| 94 | + dangling = [ |
| 95 | + s for s in suggestions if s.get("message", "").startswith("[post-prune] dangling route:") |
| 96 | + ] |
| 97 | + assert not dangling, ( |
| 98 | + f"Recipe '{recipe_name}' on backend '{backend_name}' has dangling routes: " |
| 99 | + + "; ".join(s.get("message", "") for s in dangling) |
| 100 | + ) |
| 101 | + |
| 102 | + backend_compat_errors = [ |
| 103 | + s |
| 104 | + for s in suggestions |
| 105 | + if s.get("rule") == "backend-incompatible-skill" and s.get("severity") == Severity.ERROR |
| 106 | + ] |
| 107 | + assert not backend_compat_errors, ( |
| 108 | + f"Recipe '{recipe_name}' on backend '{backend_name}' has " |
| 109 | + f"backend-incompatible-skill errors: " |
| 110 | + + "; ".join(s.get("message", "") for s in backend_compat_errors) |
| 111 | + ) |
| 112 | + |
| 113 | + |
| 114 | +def test_declared_unsupported_orphan_check() -> None: |
| 115 | + """Every DECLARED_UNSUPPORTED entry must reference a live recipe and backend.""" |
| 116 | + recipe_names = frozenset(_ALL_RECIPE_NAMES) |
| 117 | + backend_names = frozenset(BACKEND_REGISTRY.keys()) |
| 118 | + for recipe_name, backend_name in DECLARED_UNSUPPORTED: |
| 119 | + assert recipe_name in recipe_names, ( |
| 120 | + f"DECLARED_UNSUPPORTED entry ({recipe_name!r}, {backend_name!r}) " |
| 121 | + f"references unknown recipe {recipe_name!r}. " |
| 122 | + f"Valid: {sorted(recipe_names)}" |
| 123 | + ) |
| 124 | + assert backend_name in backend_names, ( |
| 125 | + f"DECLARED_UNSUPPORTED entry ({recipe_name!r}, {backend_name!r}) " |
| 126 | + f"references unknown backend {backend_name!r}. " |
| 127 | + f"Valid: {sorted(backend_names)}" |
| 128 | + ) |
| 129 | + |
| 130 | + |
| 131 | +def test_matrix_collection_count() -> None: |
| 132 | + """Matrix size = recipes x backends - declared unsupported.""" |
| 133 | + expected = len(_ALL_RECIPE_NAMES) * len(_BACKEND_NAMES) - len(DECLARED_UNSUPPORTED) |
| 134 | + assert len(_MATRIX_IDS) == expected, ( |
| 135 | + f"Matrix size mismatch: got {len(_MATRIX_IDS)}, " |
| 136 | + f"expected {len(_ALL_RECIPE_NAMES)} recipes x {len(_BACKEND_NAMES)} backends " |
| 137 | + f"- {len(DECLARED_UNSUPPORTED)} unsupported = {expected}" |
| 138 | + ) |
0 commit comments