From 62dd30367cfb65e8b905a58a75926a0b6829f988 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 26 Jun 2026 23:16:37 -0700 Subject: [PATCH 1/3] test: add discipline delivery channel matrix contract Adds tests/contracts/test_discipline_delivery_matrix.py encoding the session-type x backend discipline-delivery matrix as a parametrized 8-case contract suite. Each combination asserts the PRIMARY delivery channel is populated via the correct builder (--append-system-prompt for Claude, developer_instructions= for Codex) and that SESSION_TYPE is correctly threaded through env for headless variants. Sous-chef composition is verified to flow into orchestrator and open-kitchen prompts but NOT fleet dispatch. Updates tests/contracts/AGENTS.md file registry. --- tests/contracts/AGENTS.md | 1 + .../test_discipline_delivery_matrix.py | 177 ++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 tests/contracts/test_discipline_delivery_matrix.py diff --git a/tests/contracts/AGENTS.md b/tests/contracts/AGENTS.md index 5ff65e8ac..283fa6ea1 100644 --- a/tests/contracts/AGENTS.md +++ b/tests/contracts/AGENTS.md @@ -26,6 +26,7 @@ Protocol satisfaction, package gateway, and skill contract compliance tests. | `test_config_field_coverage.py` | REQ-CONFIG-001: every sub-config dataclass field must be referenced in from_dynaconf | | `test_core_public_api_surface.py` | Validates that every symbol in autoskillit.core.__all__ is importable via the public gateway | | `test_diagnose_ci_steps.py` | Contract tests for diagnose-ci SKILL.md step numbering and cross-reference integrity | +| `test_discipline_delivery_matrix.py` | Contract: session-type x backend discipline delivery channel matrix — interactive primary channel, headless prompt + completion marker, skill prompt, and sous-chef composition | | `test_docstring_skill_prefix.py` | Contract: source files must not use /autoskillit: prefix for skills_extended skills | | `test_environment_setup_design_contracts.py` | Contract tests verifying the environment-setup skill design doc completeness | | `test_ephemeral_skill_namespace.py` | Contract: ephemeral SKILL.md bodies use the correct namespace for the session delivery channel | diff --git a/tests/contracts/test_discipline_delivery_matrix.py b/tests/contracts/test_discipline_delivery_matrix.py new file mode 100644 index 000000000..e1219b0de --- /dev/null +++ b/tests/contracts/test_discipline_delivery_matrix.py @@ -0,0 +1,177 @@ +"""Discipline delivery channel matrix contract tests. + +Session-type x backend parametrized tests asserting that each combination's +PRIMARY delivery channel is populated via the correct builder. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.core import ( + SESSION_TYPE_ENV_VAR, + SESSION_TYPE_FLEET, + SESSION_TYPE_ORCHESTRATOR, + SESSION_TYPE_SKILL, + DirectInstall, +) +from autoskillit.execution.backends import BACKEND_REGISTRY +from autoskillit.execution.backends.claude import ClaudeCodeBackend +from autoskillit.execution.backends.codex import CodexBackend # noqa: F401 + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] + +_SESSION_TYPE_VARIANTS = 4 # fleet, orchestrator-interactive, orchestrator-headless, skill + +DELIVERY_CHANNEL_CASES: list[tuple[str, str]] = [ + ("fleet", "claude-code"), + ("fleet", "codex"), + ("orchestrator-interactive", "claude-code"), + ("orchestrator-interactive", "codex"), + ("orchestrator-headless", "claude-code"), + ("orchestrator-headless", "codex"), + ("skill", "claude-code"), + ("skill", "codex"), +] + + +def test_delivery_channel_cases_count() -> None: + assert len(DELIVERY_CHANNEL_CASES) == _SESSION_TYPE_VARIANTS * len(BACKEND_REGISTRY) + + +def _assert_interactive_primary_channel(backend, spec) -> None: + """Assert the interactive primary delivery channel is populated.""" + if isinstance(backend, ClaudeCodeBackend): + assert "--append-system-prompt" in spec.cmd + else: + assert any("developer_instructions=" in arg for arg in spec.cmd) + assert "exec" not in spec.cmd + + +class TestFleetInteractive: + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_primary_channel_populated(self, backend) -> None: + spec = backend.build_interactive_cmd( + system_prompt="Fleet discipline prompt", + env_extras={SESSION_TYPE_ENV_VAR: SESSION_TYPE_FLEET}, + ) + _assert_interactive_primary_channel(backend, spec) + + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_session_type_fleet_in_env(self, backend) -> None: + spec = backend.build_interactive_cmd( + system_prompt="Fleet discipline prompt", + env_extras={SESSION_TYPE_ENV_VAR: SESSION_TYPE_FLEET}, + ) + assert spec.env.get(SESSION_TYPE_ENV_VAR) == SESSION_TYPE_FLEET + + +class TestOrchestratorInteractive: + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_primary_channel_populated(self, backend) -> None: + spec = backend.build_interactive_cmd( + system_prompt="Orchestrator discipline prompt", + ) + _assert_interactive_primary_channel(backend, spec) + + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_no_session_type_assertion(self, backend) -> None: + spec = backend.build_interactive_cmd( + system_prompt="Orchestrator discipline prompt", + ) + assert not spec.env.get(SESSION_TYPE_ENV_VAR, "") + + +class TestOrchestratorHeadless: + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_orchestrator_prompt_non_empty(self, backend) -> None: + spec = backend.build_food_truck_cmd( + orchestrator_prompt="Run the pipeline", + plugin_source=DirectInstall(plugin_dir=Path("/tmp")), + cwd="/tmp", + completion_marker="%%DONE%%", + ) + assert any("Run the pipeline" in arg for arg in spec.cmd) + assert any("%%DONE%%" in arg for arg in spec.cmd) + + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_session_type_orchestrator_in_env(self, backend) -> None: + spec = backend.build_food_truck_cmd( + orchestrator_prompt="Run the pipeline", + plugin_source=DirectInstall(plugin_dir=Path("/tmp")), + cwd="/tmp", + completion_marker="%%DONE%%", + ) + assert spec.env.get(SESSION_TYPE_ENV_VAR) == SESSION_TYPE_ORCHESTRATOR + + +class TestSkillSession: + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_positional_prompt_non_empty(self, backend) -> None: + spec = backend.build_skill_session_cmd("/investigate foo", "/tmp") + assert any("investigate" in arg for arg in spec.cmd) + + @pytest.mark.parametrize( + "backend", + [ClaudeCodeBackend(), CodexBackend()], + ids=["claude-code", "codex"], + ) + def test_session_type_skill_in_env(self, backend) -> None: + spec = backend.build_skill_session_cmd("/investigate foo", "/tmp") + assert spec.env.get(SESSION_TYPE_ENV_VAR) == SESSION_TYPE_SKILL + + +class TestSousChefDelivery: + def test_sous_chef_in_orchestrator_prompt(self) -> None: + from autoskillit.cli._prompts import _build_orchestrator_prompt, _read_full_sous_chef + + sous_chef = _read_full_sous_chef() + assert sous_chef, "_read_full_sous_chef must return non-empty content" + prompt = _build_orchestrator_prompt("test-recipe", "mcp__autoskillit__") + assert sous_chef[:80] in prompt + + def test_sous_chef_in_open_kitchen_prompt(self) -> None: + from autoskillit.cli._prompts import _build_open_kitchen_prompt, _read_full_sous_chef + + sous_chef = _read_full_sous_chef() + assert sous_chef, "_read_full_sous_chef must return non-empty content" + prompt = _build_open_kitchen_prompt("mcp__autoskillit__") + assert sous_chef[:80] in prompt + + def test_sous_chef_not_in_fleet_dispatch_prompt(self) -> None: + from autoskillit.cli._prompts import _build_fleet_dispatch_prompt, _read_full_sous_chef + + sous_chef = _read_full_sous_chef() + assert sous_chef, "_read_full_sous_chef must return non-empty content" + prompt = _build_fleet_dispatch_prompt("mcp__autoskillit__") + assert sous_chef[:80] not in prompt From 7622a8cf68487a9ced2bbf16fcbe6cd25727c5a0 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 26 Jun 2026 23:38:01 -0700 Subject: [PATCH 2/3] fix(review): remove stale DELIVERY_CHANNEL_CASES matrix and dead count test Removes the hardcoded _SESSION_TYPE_VARIANTS=4 integer, the DELIVERY_CHANNEL_CASES list that used freeform string literals instead of SESSION_TYPE_* constants, and the self-referential test_delivery_channel_cases_count meta-test whose DELIVERY_CHANNEL_CASES source was never consumed by any @pytest.mark.parametrize decorator. Adding a backend to BACKEND_REGISTRY would have failed this count test while leaving the parametrize lists in each class silently stale, creating false coverage signal. Also removes the now-unused BACKEND_REGISTRY import. Co-Authored-By: Claude Sonnet 4.6 --- .../test_discipline_delivery_matrix.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/contracts/test_discipline_delivery_matrix.py b/tests/contracts/test_discipline_delivery_matrix.py index e1219b0de..f044e6028 100644 --- a/tests/contracts/test_discipline_delivery_matrix.py +++ b/tests/contracts/test_discipline_delivery_matrix.py @@ -17,29 +17,11 @@ SESSION_TYPE_SKILL, DirectInstall, ) -from autoskillit.execution.backends import BACKEND_REGISTRY from autoskillit.execution.backends.claude import ClaudeCodeBackend from autoskillit.execution.backends.codex import CodexBackend # noqa: F401 pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] -_SESSION_TYPE_VARIANTS = 4 # fleet, orchestrator-interactive, orchestrator-headless, skill - -DELIVERY_CHANNEL_CASES: list[tuple[str, str]] = [ - ("fleet", "claude-code"), - ("fleet", "codex"), - ("orchestrator-interactive", "claude-code"), - ("orchestrator-interactive", "codex"), - ("orchestrator-headless", "claude-code"), - ("orchestrator-headless", "codex"), - ("skill", "claude-code"), - ("skill", "codex"), -] - - -def test_delivery_channel_cases_count() -> None: - assert len(DELIVERY_CHANNEL_CASES) == _SESSION_TYPE_VARIANTS * len(BACKEND_REGISTRY) - def _assert_interactive_primary_channel(backend, spec) -> None: """Assert the interactive primary delivery channel is populated.""" From 510af604bd72e8ef2e7772206a8c761a1026a07a Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 26 Jun 2026 23:38:13 -0700 Subject: [PATCH 3/3] fix(review): remove vacuous exec-not-in-cmd assertion from Codex interactive check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_interactive_cmd never includes the literal "exec" token — only headless codex_exec_base commands do. The assertion was always trivially true and could not detect regressions where the wrong builder variant was used. The preceding developer_instructions= check is sufficient to verify the interactive primary channel. Co-Authored-By: Claude Sonnet 4.6 --- tests/contracts/test_discipline_delivery_matrix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/contracts/test_discipline_delivery_matrix.py b/tests/contracts/test_discipline_delivery_matrix.py index f044e6028..b29d1e9aa 100644 --- a/tests/contracts/test_discipline_delivery_matrix.py +++ b/tests/contracts/test_discipline_delivery_matrix.py @@ -29,7 +29,6 @@ def _assert_interactive_primary_channel(backend, spec) -> None: assert "--append-system-prompt" in spec.cmd else: assert any("developer_instructions=" in arg for arg in spec.cmd) - assert "exec" not in spec.cmd class TestFleetInteractive: