From 46ad9ec540badc7e26766c0c99325bfa901bc9d0 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 11 Jun 2026 08:34:17 -0700 Subject: [PATCH 1/3] feat(orchestrator+fleet): halt on degraded tool responses (success:false + empty content) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an explicit catch-all halt rule to both the cook-session orchestrator prompt and the fleet food-truck prompt that declares any tool response with success:false AND absent/null/empty content as a degraded state. The cook orchestrator must print user_visible_message, refuse to improvise recovery, and halt. The fleet prompt must emit the sentinel with reason="degraded_tool_response" and stop. The canonical sentinel contract in _type_dispatch_identity.py now also includes the new reason value as the IL-0 source of truth. Both prompt builders get corresponding contract tests. - src/autoskillit/core/types/_type_dispatch_identity.py: add "degraded_tool_response" to the L3 sentinel contract reason enum. - src/autoskillit/cli/_prompts_orchestrator.py: add FAILURE PREDICATE — DEGRADED TOOL RESPONSE between the open_kitchen predicate and CONTEXT LIMIT ROUTING. - src/autoskillit/fleet/_prompts.py: add FAILURE PREDICATE — DEGRADED RESPONSE between the open_kitchen predicate and TWO FAILURE TIERS, and extend the Section 8 reason enum to include the new value. - tests/cli/test_orchestrator_prompt_contract.py: new TestDegradedToolResponsePredicate class — presence, content, ordering. - tests/fleet/test_food_truck_prompt.py: three new module-level tests for the degraded halt rule, empty-content trigger, and reason enum. --- src/autoskillit/cli/_prompts_orchestrator.py | 10 ++++ .../core/types/_type_dispatch_identity.py | 2 +- src/autoskillit/fleet/_prompts.py | 9 +++- .../cli/test_orchestrator_prompt_contract.py | 41 ++++++++++++++ tests/fleet/test_food_truck_prompt.py | 54 +++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/cli/_prompts_orchestrator.py b/src/autoskillit/cli/_prompts_orchestrator.py index 21c66c39f1..37ad3b917b 100644 --- a/src/autoskillit/cli/_prompts_orchestrator.py +++ b/src/autoskillit/cli/_prompts_orchestrator.py @@ -163,6 +163,16 @@ def _build_orchestrator_prompt( 2. DO NOT call AskUserQuestion. 3. End the session with a final text response. +FAILURE PREDICATE — DEGRADED TOOL RESPONSE: + If ANY tool response contains `"success": false` AND the `content` field is + absent, null, or an empty string — this is a degraded state. + 1. Extract and print the value of "user_visible_message" from the + response (fall back to the raw response text if the field is missing). + 2. DO NOT call AskUserQuestion. + 3. Do not improvise a recovery path. Do not retry the tool without + explicit instructions from the recipe step. + 4. End the session with a final text response. + CONTEXT LIMIT ROUTING — run_skill only (check BEFORE on_failure): - When run_skill returns "success: False" AND "needs_retry: true" AND "retry_reason: resume": - Check "subtype" to discriminate the termination cause: diff --git a/src/autoskillit/core/types/_type_dispatch_identity.py b/src/autoskillit/core/types/_type_dispatch_identity.py index e1f7fc61e3..edaea263c5 100644 --- a/src/autoskillit/core/types/_type_dispatch_identity.py +++ b/src/autoskillit/core/types/_type_dispatch_identity.py @@ -32,7 +32,7 @@ def _build_sentinel_contract(dispatch_id: str, short: str) -> str: Fields: - success: true if all mandatory steps completed without unresolved failures - reason: "completed", "failed", "fleet_quota_exhausted", "timeout", - "open_kitchen_failed", "missing_on_failure" + "open_kitchen_failed", "missing_on_failure", "degraded_tool_response" - summary: One-line description of what happened The sentinel markers ---l3-result::{dispatch_id}--- and ---end-l3-result::{dispatch_id}--- diff --git a/src/autoskillit/fleet/_prompts.py b/src/autoskillit/fleet/_prompts.py index 810a23d0bf..ee0a4f2f0f 100644 --- a/src/autoskillit/fleet/_prompts.py +++ b/src/autoskillit/fleet/_prompts.py @@ -206,6 +206,13 @@ def _build_food_truck_prompt( 1. Emit the sentinel block with success=false and reason="open_kitchen_failed". 2. End the session. +FAILURE PREDICATE — DEGRADED RESPONSE: + If ANY tool response contains success=false AND the content field is + absent, null, or an empty string — this is a degraded state. + 1. Emit the sentinel block with success=false and reason="degraded_tool_response". + 2. Stop. Do not improvise a recovery path, do not route to on_failure, + and do not retry the tool. + TWO FAILURE TIERS FOR PREDICATE-FORMAT STEPS: - Tool-level failure (run_skill returns "success: False"): Follow on_failure. This fires BEFORE any result object exists. on_result conditions are NOT evaluated. @@ -340,7 +347,7 @@ def _build_food_truck_prompt( Fields: - success: true if all mandatory steps completed without unresolved failures - reason: "completed", "failed", "quota_exhausted", "timeout", - "open_kitchen_failed", "missing_on_failure" + "open_kitchen_failed", "missing_on_failure", "degraded_tool_response" - summary: One-line description of what happened{extra_fields_docs} The sentinel markers ---l3-result::{dispatch_id}--- and ---end-l3-result::{dispatch_id}--- diff --git a/tests/cli/test_orchestrator_prompt_contract.py b/tests/cli/test_orchestrator_prompt_contract.py index e9954acd0a..d15d83a413 100644 --- a/tests/cli/test_orchestrator_prompt_contract.py +++ b/tests/cli/test_orchestrator_prompt_contract.py @@ -51,6 +51,47 @@ def test_open_kitchen_predicate_uses_json_field_not_substring(self): assert "ingredients_table" in section +class TestDegradedToolResponsePredicate: + """Guards for the FAILURE PREDICATE — DEGRADED TOOL RESPONSE block.""" + + def _get_section(self) -> str: + prompt = _get_prompt() + start = prompt.index("FAILURE PREDICATE — DEGRADED TOOL RESPONSE") + end = prompt.index("CONTEXT LIMIT ROUTING", start) + return prompt[start:end] + + def test_prompt_contains_degraded_tool_response_section(self): + prompt = _get_prompt() + ok_idx = prompt.index("FAILURE PREDICATE — open_kitchen") + deg_idx = prompt.index("FAILURE PREDICATE — DEGRADED TOOL RESPONSE") + ctx_idx = prompt.index("CONTEXT LIMIT ROUTING") + assert ok_idx < deg_idx < ctx_idx, ( + "Degraded predicate must appear after open_kitchen and before CONTEXT LIMIT ROUTING" + ) + + def test_degraded_predicate_covers_success_false_empty_content(self): + section = self._get_section() + assert "success" in section + assert "false" in section + assert any(word in section for word in ("absent", "null", "empty")), ( + "Section must reference absent, null, or empty content" + ) + + def test_degraded_predicate_prohibits_improvisation(self): + section = self._get_section() + lower = section.lower() + assert any( + phrase in lower + for phrase in ( + "do not improvise", + "do not retry", + "do not attempt", + "never", + "halt", + ) + ), "Section must prohibit improvisation or retry" + + class TestStep0ToolPredicateCoverage: """Every tool referenced in STEP 0 must have a failure predicate or shared rule.""" diff --git a/tests/fleet/test_food_truck_prompt.py b/tests/fleet/test_food_truck_prompt.py index cf4402aff5..4d9c980b40 100644 --- a/tests/fleet/test_food_truck_prompt.py +++ b/tests/fleet/test_food_truck_prompt.py @@ -174,3 +174,57 @@ def test_food_truck_open_kitchen_predicate_uses_json_field_not_substring(): section = prompt[idx : idx + 500] assert "--- INGREDIENTS TABLE ---" not in section assert "ingredients_table" in section + + +def test_food_truck_prompt_has_degraded_response_halt_rule(): + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + assert "FAILURE PREDICATE — DEGRADED RESPONSE" in prompt + assert 'reason="degraded_tool_response"' in prompt + + +def test_food_truck_prompt_degraded_rule_triggers_sentinel_on_empty_content(): + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + start = prompt.index("FAILURE PREDICATE — DEGRADED RESPONSE") + end = prompt.index("TWO FAILURE TIERS", start) + section = prompt[start:end] + assert "success" in section + assert "false" in section + assert any(word in section for word in ("absent", "null", "empty")), ( + "Rule must reference absent, null, or empty content" + ) + + +def test_food_truck_prompt_degraded_reason_in_reason_enum(): + prompt = _build_food_truck_prompt( + recipe="test-recipe", + task="Test task", + ingredients={}, + mcp_prefix=DIRECT_PREFIX, + dispatch_id="test-dispatch", + campaign_id="test-campaign", + l3_timeout_sec=300, + ) + fields_idx = prompt.index("Fields:") + reason_region = prompt[fields_idx : fields_idx + 300] + assert "degraded_tool_response" in reason_region, ( + "degraded_tool_response must appear in Section 8 reason enum" + ) + ok_idx = prompt.index("FAILURE PREDICATE — open_kitchen") + deg_idx = prompt.index("FAILURE PREDICATE — DEGRADED RESPONSE") + assert ok_idx != deg_idx, "Degraded and open_kitchen predicates must be distinct blocks" From 19cbaf8efa9bed8247d3f1e2b0ddccd80f20d975 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 11 Jun 2026 08:58:19 -0700 Subject: [PATCH 2/3] fix(review): add explicit on_failure suppression to orchestrator DEGRADED TOOL RESPONSE predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the fleet prompt's 'do not route to on_failure' prohibition in the orchestrator predicate — the asymmetry meant the orchestrator could still attempt on_failure handling on a degraded response. --- src/autoskillit/cli/_prompts_orchestrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/cli/_prompts_orchestrator.py b/src/autoskillit/cli/_prompts_orchestrator.py index 37ad3b917b..62f78f741e 100644 --- a/src/autoskillit/cli/_prompts_orchestrator.py +++ b/src/autoskillit/cli/_prompts_orchestrator.py @@ -169,8 +169,8 @@ def _build_orchestrator_prompt( 1. Extract and print the value of "user_visible_message" from the response (fall back to the raw response text if the field is missing). 2. DO NOT call AskUserQuestion. - 3. Do not improvise a recovery path. Do not retry the tool without - explicit instructions from the recipe step. + 3. Do not improvise a recovery path. Do not route to on_failure. + Do not retry the tool without explicit instructions from the recipe step. 4. End the session with a final text response. CONTEXT LIMIT ROUTING — run_skill only (check BEFORE on_failure): From 1d0d3b4674d5fa410ed4e2d1fae25075fb14ed71 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 11 Jun 2026 08:58:28 -0700 Subject: [PATCH 3/3] fix(review): anchor fleet test Fields: search to SECTION 8 header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prompt.index('Fields:') finds the first occurrence — anchor to '--- SECTION 8:' first so future additional Fields: sections in the prompt don't silently cause the assertion to check the wrong region. --- tests/fleet/test_food_truck_prompt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fleet/test_food_truck_prompt.py b/tests/fleet/test_food_truck_prompt.py index 4d9c980b40..439627303a 100644 --- a/tests/fleet/test_food_truck_prompt.py +++ b/tests/fleet/test_food_truck_prompt.py @@ -220,7 +220,7 @@ def test_food_truck_prompt_degraded_reason_in_reason_enum(): campaign_id="test-campaign", l3_timeout_sec=300, ) - fields_idx = prompt.index("Fields:") + fields_idx = prompt.index("Fields:", prompt.index("--- SECTION 8:")) reason_region = prompt[fields_idx : fields_idx + 300] assert "degraded_tool_response" in reason_region, ( "degraded_tool_response must appear in Section 8 reason enum"