Skip to content

Commit e0a9054

Browse files
test: adversarial overlap fixtures for tool dispatch
Add test_tool_dispatch_adversarial.py with overlap JSON for every ORDERING_INVARIANTS pair, checking winners through _parse_tool_result. Share ORDERING_INVARIANT_IDS with the ordering test module. Tighten the sync guard and add a monkeypatch case that shows plan/file_write misclassification when dispatch order is wrong.
1 parent a725bd2 commit e0a9054

2 files changed

Lines changed: 149 additions & 6 deletions

File tree

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
"""Behavioral adversarial fixtures for ``_TOOL_RESULT_DISPATCH`` predicate overlap.
2+
3+
Structural tuple-position guards live in ``test_tool_dispatch_ordering.py``.
4+
These tests construct ``toolUseResult`` JSON that satisfies multiple predicates
5+
and assert the classified winner via ``_parse_tool_result`` (first match wins).
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from collections.abc import Callable
11+
12+
import pytest
13+
14+
from tests.test_tool_dispatch_ordering import ORDERING_INVARIANT_IDS, ORDERING_INVARIANTS
15+
from utils import tool_dispatch
16+
from utils.tool_dispatch import _TOOL_RESULT_DISPATCH, _parse_tool_result
17+
18+
# Overlap blobs: keys chosen so both predicates in each ORDERING_INVARIANTS pair match.
19+
PLAN_FILE_WRITE_OVERLAP: dict[str, object] = {
20+
"plan": {"name": "sprint-plan", "steps": ["index", "search", "render"]},
21+
"filePath": ".cursor/plans/week28.md",
22+
"content": "Plan body that would also satisfy file_write.",
23+
}
24+
25+
TASK_MESSAGE_RETRIEVAL_OVERLAP: dict[str, object] = {
26+
"task_id": "task-overlap-retrieval",
27+
"message": "polling retrieval",
28+
"retrieval_status": "found",
29+
"task": {"task_id": "task-overlap-retrieval", "description": "subagent scan"},
30+
}
31+
32+
TASK_MESSAGE_COMPLETED_OVERLAP: dict[str, object] = {
33+
"agentId": "agent-overlap-completed",
34+
"totalDurationMs": 3200,
35+
"totalTokens": 900,
36+
"status": "completed",
37+
"message": "task finished",
38+
}
39+
40+
TASK_MESSAGE_ASYNC_OVERLAP: dict[str, object] = {
41+
"agentId": "agent-overlap-async",
42+
"isAsync": True,
43+
"description": "explore auth handlers",
44+
"message": "task launched",
45+
}
46+
47+
# Narrow retrieval shape: only the downstream predicate matches (no task_id/message).
48+
TASK_RETRIEVAL_NARROW: dict[str, object] = {
49+
"retrieval_status": "pending",
50+
"task": {"task_id": "task-narrow", "description": "wait for result"},
51+
}
52+
53+
AssertWinner = Callable[[dict[str, object]], None]
54+
55+
56+
def _assert_plan_beats_file_write(result: dict[str, object]) -> None:
57+
assert result["result_type"] == "plan"
58+
assert result.get("file_path") == ".cursor/plans/week28.md"
59+
60+
61+
def _assert_task_message_beats_retrieval(result: dict[str, object]) -> None:
62+
assert result["result_type"] == "task"
63+
assert result.get("task_id") == "task-overlap-retrieval"
64+
assert "retrieval_status" not in result
65+
66+
67+
def _assert_task_message_beats_completed(result: dict[str, object]) -> None:
68+
assert result["result_type"] == "task"
69+
assert result.get("agent_id") is None
70+
assert result.get("total_duration_ms") is None
71+
72+
73+
def _assert_task_message_beats_async(result: dict[str, object]) -> None:
74+
assert result["result_type"] == "task"
75+
assert result.get("agent_id") is None
76+
assert result.get("description") is None
77+
78+
79+
_INVARIANT_BEHAVIOR: dict[str, tuple[dict[str, object], AssertWinner]] = {
80+
"plan_before_file_write": (PLAN_FILE_WRITE_OVERLAP, _assert_plan_beats_file_write),
81+
"task_message_before_task_retrieval": (
82+
TASK_MESSAGE_RETRIEVAL_OVERLAP,
83+
_assert_task_message_beats_retrieval,
84+
),
85+
"task_message_before_task_completed": (
86+
TASK_MESSAGE_COMPLETED_OVERLAP,
87+
_assert_task_message_beats_completed,
88+
),
89+
"task_message_before_task_async": (
90+
TASK_MESSAGE_ASYNC_OVERLAP,
91+
_assert_task_message_beats_async,
92+
),
93+
}
94+
95+
@pytest.mark.parametrize(
96+
"fixture_id",
97+
ORDERING_INVARIANT_IDS,
98+
)
99+
def test_adversarial_overlap_classifies_documented_winner(fixture_id: str) -> None:
100+
blob, assert_winner = _INVARIANT_BEHAVIOR[fixture_id]
101+
result = _parse_tool_result(blob)
102+
assert result is not None
103+
assert_winner(result)
104+
105+
106+
def test_task_retrieval_narrow_shape_without_task_message_keys() -> None:
107+
"""Boundary: narrow retrieval wins when broad task_message keys are absent."""
108+
result = _parse_tool_result(TASK_RETRIEVAL_NARROW)
109+
assert result is not None
110+
assert result["result_type"] == "task"
111+
assert result.get("retrieval_status") == "pending"
112+
assert result.get("task_id") == "task-narrow"
113+
114+
115+
def test_inverted_plan_file_write_dispatch_misclassifies_overlap(
116+
monkeypatch: pytest.MonkeyPatch,
117+
) -> None:
118+
"""Regression: swapping plan below file_write flips the overlap winner."""
119+
table = list(_TOOL_RESULT_DISPATCH)
120+
plan_idx = next(
121+
i for i, (pred, _) in enumerate(table) if pred.__name__ == "is_plan_tool_result"
122+
)
123+
write_idx = next(
124+
i for i, (pred, _) in enumerate(table) if pred.__name__ == "is_file_write_tool_result"
125+
)
126+
table[plan_idx], table[write_idx] = table[write_idx], table[plan_idx]
127+
monkeypatch.setattr(tool_dispatch, "_TOOL_RESULT_DISPATCH", tuple(table))
128+
129+
result = _parse_tool_result(PLAN_FILE_WRITE_OVERLAP)
130+
assert result is not None
131+
assert result["result_type"] == "file_write"
132+
assert result.get("file_path") == ".cursor/plans/week28.md"
133+
with pytest.raises(AssertionError):
134+
_assert_plan_beats_file_write(result)
135+
136+
137+
def test_ordering_invariants_have_adversarial_coverage() -> None:
138+
"""Every ORDERING_INVARIANTS row has a behavioral fixture (keeps tables in sync)."""
139+
assert len(ORDERING_INVARIANTS) == len(ORDERING_INVARIANT_IDS)
140+
assert len(ORDERING_INVARIANT_IDS) == len(_INVARIANT_BEHAVIOR)
141+
assert set(_INVARIANT_BEHAVIOR.keys()) == set(ORDERING_INVARIANT_IDS)

tests/test_tool_dispatch_ordering.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@
4444
),
4545
]
4646

47+
ORDERING_INVARIANT_IDS = [
48+
"plan_before_file_write",
49+
"task_message_before_task_retrieval",
50+
"task_message_before_task_completed",
51+
"task_message_before_task_async",
52+
]
53+
4754

4855
def _predicate_index(predicate: Predicate) -> int:
4956
for i, entry in enumerate(_TOOL_RESULT_DISPATCH):
@@ -57,12 +64,7 @@ def _predicate_index(predicate: Predicate) -> int:
5764
@pytest.mark.parametrize(
5865
"before,after,reason",
5966
ORDERING_INVARIANTS,
60-
ids=[
61-
"plan_before_file_write",
62-
"task_message_before_task_retrieval",
63-
"task_message_before_task_completed",
64-
"task_message_before_task_async",
65-
],
67+
ids=ORDERING_INVARIANT_IDS,
6668
)
6769
def test_tool_dispatch_ordering_invariant(
6870
before: Predicate,

0 commit comments

Comments
 (0)