Skip to content

Commit 1f9ae2e

Browse files
piratepirate
committed
new model shape tests
1 parent c46c1ef commit 1f9ae2e

File tree

1 file changed

+288
-0
lines changed

1 file changed

+288
-0
lines changed

‎tests/test_session_model_config_params.py‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
# Manually maintained tests for model config request serialization.
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import json
7+
from typing import Any, cast
8+
9+
import httpx
10+
import pytest
11+
from respx import MockRouter
12+
from respx.models import Call
13+
14+
from stagehand import Stagehand, AsyncStagehand
15+
16+
base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
17+
18+
MODEL_STRING = "openai/gpt-5-nano"
19+
MODEL_OBJECT_INPUT = {
20+
"model_name": "openai/gpt-5-nano",
21+
"api_key": "sk-some-openai-api-key",
22+
"base_url": "https://api.openai.com/v1",
23+
"headers": {"x-foo": "bar"},
24+
"provider": "openai",
25+
}
26+
MODEL_OBJECT_WIRE = {
27+
"modelName": "openai/gpt-5-nano",
28+
"apiKey": "sk-some-openai-api-key",
29+
"baseURL": "https://api.openai.com/v1",
30+
"headers": {"x-foo": "bar"},
31+
"provider": "openai",
32+
}
33+
34+
35+
def _mock_start_route(respx_mock: MockRouter, session_id: str) -> None:
36+
respx_mock.post("/v1/sessions/start").mock(
37+
return_value=httpx.Response(
38+
200,
39+
json={"success": True, "data": {"available": True, "sessionId": session_id}},
40+
)
41+
)
42+
43+
44+
def _mock_observe_route(respx_mock: MockRouter, session_id: str):
45+
return respx_mock.post(f"/v1/sessions/{session_id}/observe").mock(
46+
return_value=httpx.Response(
47+
200,
48+
json={
49+
"success": True,
50+
"data": {"result": [{"description": "Click submit", "selector": "button[type=submit]"}]},
51+
},
52+
)
53+
)
54+
55+
56+
def _mock_extract_route(respx_mock: MockRouter, session_id: str):
57+
return respx_mock.post(f"/v1/sessions/{session_id}/extract").mock(
58+
return_value=httpx.Response(
59+
200,
60+
json={"success": True, "data": {"result": {"value": "ok"}}},
61+
)
62+
)
63+
64+
65+
def _mock_execute_route(respx_mock: MockRouter, session_id: str):
66+
return respx_mock.post(f"/v1/sessions/{session_id}/agentExecute").mock(
67+
return_value=httpx.Response(
68+
200,
69+
json={
70+
"success": True,
71+
"data": {"result": {"actions": [], "completed": True, "message": "done", "success": True}},
72+
},
73+
)
74+
)
75+
76+
77+
def _request_json(call: Call) -> dict[str, object]:
78+
return cast(dict[str, object], json.loads(call.request.content))
79+
80+
81+
@pytest.mark.respx(base_url=base_url)
82+
def test_session_start_serializes_string_model_name(respx_mock: MockRouter, client: Stagehand) -> None:
83+
session_id = "00000000-0000-0000-0000-000000000010"
84+
start_route = respx_mock.post("/v1/sessions/start").mock(
85+
return_value=httpx.Response(
86+
200,
87+
json={"success": True, "data": {"available": True, "sessionId": session_id}},
88+
)
89+
)
90+
91+
session = client.sessions.start(model_name=MODEL_STRING)
92+
93+
assert session.id == session_id
94+
request_body = _request_json(cast(Call, start_route.calls[0]))
95+
assert request_body["modelName"] == MODEL_STRING
96+
97+
98+
@pytest.mark.respx(base_url=base_url)
99+
def test_session_observe_serializes_string_and_object_model(respx_mock: MockRouter, client: Stagehand) -> None:
100+
session_id = "00000000-0000-0000-0000-000000000011"
101+
_mock_start_route(respx_mock, session_id)
102+
observe_route = _mock_observe_route(respx_mock, session_id)
103+
104+
session = client.sessions.start(model_name=MODEL_STRING)
105+
106+
session.observe(instruction="find the submit button", options={"model": MODEL_STRING})
107+
session.observe(
108+
instruction="find the submit button",
109+
options=cast(Any, {"model": MODEL_OBJECT_INPUT}),
110+
)
111+
112+
first_request = _request_json(cast(Call, observe_route.calls[0]))
113+
second_request = _request_json(cast(Call, observe_route.calls[1]))
114+
assert cast(dict[str, object], first_request["options"])["model"] == MODEL_STRING
115+
assert cast(dict[str, object], second_request["options"])["model"] == MODEL_OBJECT_WIRE
116+
117+
118+
@pytest.mark.respx(base_url=base_url)
119+
def test_session_extract_serializes_string_and_object_model(respx_mock: MockRouter, client: Stagehand) -> None:
120+
session_id = "00000000-0000-0000-0000-000000000012"
121+
_mock_start_route(respx_mock, session_id)
122+
extract_route = _mock_extract_route(respx_mock, session_id)
123+
124+
session = client.sessions.start(model_name=MODEL_STRING)
125+
126+
session.extract(
127+
instruction="extract the result",
128+
schema={"type": "object", "properties": {"value": {"type": "string"}}},
129+
options={"model": MODEL_STRING},
130+
)
131+
session.extract(
132+
instruction="extract the result",
133+
schema={"type": "object", "properties": {"value": {"type": "string"}}},
134+
options=cast(Any, {"model": MODEL_OBJECT_INPUT}),
135+
)
136+
137+
first_request = _request_json(cast(Call, extract_route.calls[0]))
138+
second_request = _request_json(cast(Call, extract_route.calls[1]))
139+
assert cast(dict[str, object], first_request["options"])["model"] == MODEL_STRING
140+
assert cast(dict[str, object], second_request["options"])["model"] == MODEL_OBJECT_WIRE
141+
142+
143+
@pytest.mark.respx(base_url=base_url)
144+
def test_session_execute_serializes_string_and_object_models(respx_mock: MockRouter, client: Stagehand) -> None:
145+
session_id = "00000000-0000-0000-0000-000000000013"
146+
_mock_start_route(respx_mock, session_id)
147+
execute_route = _mock_execute_route(respx_mock, session_id)
148+
149+
session = client.sessions.start(model_name=MODEL_STRING)
150+
151+
session.execute(
152+
agent_config=cast(
153+
Any,
154+
{
155+
"model": MODEL_STRING,
156+
"execution_model": MODEL_OBJECT_INPUT,
157+
},
158+
),
159+
execute_options={"instruction": "click submit"},
160+
)
161+
session.execute(
162+
agent_config=cast(
163+
Any,
164+
{
165+
"model": MODEL_OBJECT_INPUT,
166+
"execution_model": MODEL_STRING,
167+
},
168+
),
169+
execute_options={"instruction": "click submit"},
170+
)
171+
172+
first_request = _request_json(cast(Call, execute_route.calls[0]))
173+
second_request = _request_json(cast(Call, execute_route.calls[1]))
174+
first_agent_config = cast(dict[str, object], first_request["agentConfig"])
175+
second_agent_config = cast(dict[str, object], second_request["agentConfig"])
176+
assert first_agent_config["model"] == MODEL_STRING
177+
assert first_agent_config["executionModel"] == MODEL_OBJECT_WIRE
178+
assert second_agent_config["model"] == MODEL_OBJECT_WIRE
179+
assert second_agent_config["executionModel"] == MODEL_STRING
180+
181+
182+
@pytest.mark.respx(base_url=base_url)
183+
async def test_async_session_start_serializes_string_model_name(
184+
respx_mock: MockRouter, async_client: AsyncStagehand
185+
) -> None:
186+
session_id = "00000000-0000-0000-0000-000000000014"
187+
start_route = respx_mock.post("/v1/sessions/start").mock(
188+
return_value=httpx.Response(
189+
200,
190+
json={"success": True, "data": {"available": True, "sessionId": session_id}},
191+
)
192+
)
193+
194+
session = await async_client.sessions.start(model_name=MODEL_STRING)
195+
196+
assert session.id == session_id
197+
request_body = _request_json(cast(Call, start_route.calls[0]))
198+
assert request_body["modelName"] == MODEL_STRING
199+
200+
201+
@pytest.mark.respx(base_url=base_url)
202+
async def test_async_session_observe_serializes_string_and_object_model(
203+
respx_mock: MockRouter, async_client: AsyncStagehand
204+
) -> None:
205+
session_id = "00000000-0000-0000-0000-000000000015"
206+
_mock_start_route(respx_mock, session_id)
207+
observe_route = _mock_observe_route(respx_mock, session_id)
208+
209+
session = await async_client.sessions.start(model_name=MODEL_STRING)
210+
211+
await session.observe(instruction="find the submit button", options={"model": MODEL_STRING})
212+
await session.observe(
213+
instruction="find the submit button",
214+
options=cast(Any, {"model": MODEL_OBJECT_INPUT}),
215+
)
216+
217+
first_request = _request_json(cast(Call, observe_route.calls[0]))
218+
second_request = _request_json(cast(Call, observe_route.calls[1]))
219+
assert cast(dict[str, object], first_request["options"])["model"] == MODEL_STRING
220+
assert cast(dict[str, object], second_request["options"])["model"] == MODEL_OBJECT_WIRE
221+
222+
223+
@pytest.mark.respx(base_url=base_url)
224+
async def test_async_session_extract_serializes_string_and_object_model(
225+
respx_mock: MockRouter, async_client: AsyncStagehand
226+
) -> None:
227+
session_id = "00000000-0000-0000-0000-000000000016"
228+
_mock_start_route(respx_mock, session_id)
229+
extract_route = _mock_extract_route(respx_mock, session_id)
230+
231+
session = await async_client.sessions.start(model_name=MODEL_STRING)
232+
233+
await session.extract(
234+
instruction="extract the result",
235+
schema={"type": "object", "properties": {"value": {"type": "string"}}},
236+
options={"model": MODEL_STRING},
237+
)
238+
await session.extract(
239+
instruction="extract the result",
240+
schema={"type": "object", "properties": {"value": {"type": "string"}}},
241+
options=cast(Any, {"model": MODEL_OBJECT_INPUT}),
242+
)
243+
244+
first_request = _request_json(cast(Call, extract_route.calls[0]))
245+
second_request = _request_json(cast(Call, extract_route.calls[1]))
246+
assert cast(dict[str, object], first_request["options"])["model"] == MODEL_STRING
247+
assert cast(dict[str, object], second_request["options"])["model"] == MODEL_OBJECT_WIRE
248+
249+
250+
@pytest.mark.respx(base_url=base_url)
251+
async def test_async_session_execute_serializes_string_and_object_models(
252+
respx_mock: MockRouter, async_client: AsyncStagehand
253+
) -> None:
254+
session_id = "00000000-0000-0000-0000-000000000017"
255+
_mock_start_route(respx_mock, session_id)
256+
execute_route = _mock_execute_route(respx_mock, session_id)
257+
258+
session = await async_client.sessions.start(model_name=MODEL_STRING)
259+
260+
await session.execute(
261+
agent_config=cast(
262+
Any,
263+
{
264+
"model": MODEL_STRING,
265+
"execution_model": MODEL_OBJECT_INPUT,
266+
},
267+
),
268+
execute_options={"instruction": "click submit"},
269+
)
270+
await session.execute(
271+
agent_config=cast(
272+
Any,
273+
{
274+
"model": MODEL_OBJECT_INPUT,
275+
"execution_model": MODEL_STRING,
276+
},
277+
),
278+
execute_options={"instruction": "click submit"},
279+
)
280+
281+
first_request = _request_json(cast(Call, execute_route.calls[0]))
282+
second_request = _request_json(cast(Call, execute_route.calls[1]))
283+
first_agent_config = cast(dict[str, object], first_request["agentConfig"])
284+
second_agent_config = cast(dict[str, object], second_request["agentConfig"])
285+
assert first_agent_config["model"] == MODEL_STRING
286+
assert first_agent_config["executionModel"] == MODEL_OBJECT_WIRE
287+
assert second_agent_config["model"] == MODEL_OBJECT_WIRE
288+
assert second_agent_config["executionModel"] == MODEL_STRING

0 commit comments

Comments
 (0)