Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/autoskillit/cli/_prompts_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,16 @@ def _build_orchestrator_prompt(
2. DO NOT call AskUserQuestion.
3. End the session with a final text response.

FAILURE PREDICATE β€” DEGRADED TOOL RESPONSE:
If ANY tool response contains `"success": false` AND the `content` field is
absent, null, or an empty string β€” this is a degraded state.
1. Extract and print the value of "user_visible_message" from the
response (fall back to the raw response text if the field is missing).
2. DO NOT call AskUserQuestion.
3. Do not improvise a recovery path. Do not route to on_failure.
Do not retry the tool without explicit instructions from the recipe step.
4. End the session with a final text response.

CONTEXT LIMIT ROUTING β€” run_skill only (check BEFORE on_failure):
- When run_skill returns "success: False" AND "needs_retry: true" AND "retry_reason: resume":
- Check "subtype" to discriminate the termination cause:
Expand Down
2 changes: 1 addition & 1 deletion src/autoskillit/core/types/_type_dispatch_identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def _build_sentinel_contract(dispatch_id: str, short: str) -> str:
Fields:
- success: true if all mandatory steps completed without unresolved failures
- reason: "completed", "failed", "fleet_quota_exhausted", "timeout",
"open_kitchen_failed", "missing_on_failure"
"open_kitchen_failed", "missing_on_failure", "degraded_tool_response"
- summary: One-line description of what happened

The sentinel markers ---l3-result::{dispatch_id}--- and ---end-l3-result::{dispatch_id}---
Expand Down
9 changes: 8 additions & 1 deletion src/autoskillit/fleet/_prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,13 @@ def _build_food_truck_prompt(
1. Emit the sentinel block with success=false and reason="open_kitchen_failed".
2. End the session.

FAILURE PREDICATE β€” DEGRADED RESPONSE:
If ANY tool response contains success=false AND the content field is
absent, null, or an empty string β€” this is a degraded state.
1. Emit the sentinel block with success=false and reason="degraded_tool_response".
2. Stop. Do not improvise a recovery path, do not route to on_failure,
and do not retry the tool.

TWO FAILURE TIERS FOR PREDICATE-FORMAT STEPS:
- Tool-level failure (run_skill returns "success: False"): Follow on_failure. This fires
BEFORE any result object exists. on_result conditions are NOT evaluated.
Expand Down Expand Up @@ -340,7 +347,7 @@ def _build_food_truck_prompt(
Fields:
- success: true if all mandatory steps completed without unresolved failures
- reason: "completed", "failed", "quota_exhausted", "timeout",
"open_kitchen_failed", "missing_on_failure"
"open_kitchen_failed", "missing_on_failure", "degraded_tool_response"
- summary: One-line description of what happened{extra_fields_docs}

The sentinel markers ---l3-result::{dispatch_id}--- and ---end-l3-result::{dispatch_id}---
Expand Down
41 changes: 41 additions & 0 deletions tests/cli/test_orchestrator_prompt_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,47 @@ def test_open_kitchen_predicate_uses_json_field_not_substring(self):
assert "ingredients_table" in section


class TestDegradedToolResponsePredicate:
"""Guards for the FAILURE PREDICATE β€” DEGRADED TOOL RESPONSE block."""

def _get_section(self) -> str:
prompt = _get_prompt()
start = prompt.index("FAILURE PREDICATE β€” DEGRADED TOOL RESPONSE")
end = prompt.index("CONTEXT LIMIT ROUTING", start)
return prompt[start:end]

def test_prompt_contains_degraded_tool_response_section(self):
prompt = _get_prompt()
ok_idx = prompt.index("FAILURE PREDICATE β€” open_kitchen")
deg_idx = prompt.index("FAILURE PREDICATE β€” DEGRADED TOOL RESPONSE")
ctx_idx = prompt.index("CONTEXT LIMIT ROUTING")
assert ok_idx < deg_idx < ctx_idx, (
"Degraded predicate must appear after open_kitchen and before CONTEXT LIMIT ROUTING"
)

def test_degraded_predicate_covers_success_false_empty_content(self):
section = self._get_section()
assert "success" in section
assert "false" in section
assert any(word in section for word in ("absent", "null", "empty")), (
"Section must reference absent, null, or empty content"
)

def test_degraded_predicate_prohibits_improvisation(self):
section = self._get_section()
lower = section.lower()
assert any(
phrase in lower
for phrase in (
"do not improvise",
"do not retry",
"do not attempt",
"never",
"halt",
)
), "Section must prohibit improvisation or retry"


class TestStep0ToolPredicateCoverage:
"""Every tool referenced in STEP 0 must have a failure predicate or shared rule."""

Expand Down
54 changes: 54 additions & 0 deletions tests/fleet/test_food_truck_prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,3 +174,57 @@ def test_food_truck_open_kitchen_predicate_uses_json_field_not_substring():
section = prompt[idx : idx + 500]
assert "--- INGREDIENTS TABLE ---" not in section
assert "ingredients_table" in section


def test_food_truck_prompt_has_degraded_response_halt_rule():
prompt = _build_food_truck_prompt(
recipe="test-recipe",
task="Test task",
ingredients={},
mcp_prefix=DIRECT_PREFIX,
dispatch_id="test-dispatch",
campaign_id="test-campaign",
l3_timeout_sec=300,
)
assert "FAILURE PREDICATE β€” DEGRADED RESPONSE" in prompt
assert 'reason="degraded_tool_response"' in prompt


def test_food_truck_prompt_degraded_rule_triggers_sentinel_on_empty_content():
prompt = _build_food_truck_prompt(
recipe="test-recipe",
task="Test task",
ingredients={},
mcp_prefix=DIRECT_PREFIX,
dispatch_id="test-dispatch",
campaign_id="test-campaign",
l3_timeout_sec=300,
)
start = prompt.index("FAILURE PREDICATE β€” DEGRADED RESPONSE")
end = prompt.index("TWO FAILURE TIERS", start)
section = prompt[start:end]
assert "success" in section
assert "false" in section
assert any(word in section for word in ("absent", "null", "empty")), (
"Rule must reference absent, null, or empty content"
)


def test_food_truck_prompt_degraded_reason_in_reason_enum():
prompt = _build_food_truck_prompt(
recipe="test-recipe",
task="Test task",
ingredients={},
mcp_prefix=DIRECT_PREFIX,
dispatch_id="test-dispatch",
campaign_id="test-campaign",
l3_timeout_sec=300,
)
fields_idx = prompt.index("Fields:", prompt.index("--- SECTION 8:"))
reason_region = prompt[fields_idx : fields_idx + 300]
assert "degraded_tool_response" in reason_region, (
"degraded_tool_response must appear in Section 8 reason enum"
)
ok_idx = prompt.index("FAILURE PREDICATE β€” open_kitchen")
deg_idx = prompt.index("FAILURE PREDICATE β€” DEGRADED RESPONSE")
assert ok_idx != deg_idx, "Degraded and open_kitchen predicates must be distinct blocks"
Loading