Skip to content

Commit 0cd9d28

Browse files
DNKYrDNKYr
committed
tests: add replay case definitions and data models
Add ReplayOp / ReplayCase Pydantic models and 10 standardized replay trajectory files covering: - 01 single-turn chat - 02 multi-turn conversation - 03 tool call (function_call + function_response) - 04 state update (write and overwrite) - 05 memory store and search - 06 summary generation and retrieval - 07 summary with event truncation - 08 error recovery (duplicate append) - 09 injected anomaly (extra event detection) - 10 injected anomaly (state corruption detection) Updates #89 RELEASE NOTES: NONE
1 parent 2fc7756 commit 0cd9d28

11 files changed

Lines changed: 588 additions & 0 deletions

‎tests/sessions/replay_cases/01_single_turn.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"case_id": "01_single_turn",
3+
"description": "Single turn: user sends a message and agent responds with plain text",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-001"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "Hello, how are you?"
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "I'm doing well, thank you for asking!"
19+
},
20+
{
21+
"op": "read_back",
22+
"expected_event_count": 2
23+
}
24+
]
25+
}

‎tests/sessions/replay_cases/02_multi_turn.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"case_id": "02_multi_turn",
3+
"description": "Multi-turn conversation: three rounds of user/assistant exchange",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-002"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "What's the weather today?"
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "It's sunny and 72 degrees."
19+
},
20+
{
21+
"op": "append_event",
22+
"author": "user",
23+
"text": "Great, I'll go for a walk."
24+
},
25+
{
26+
"op": "append_event",
27+
"author": "assistant",
28+
"text": "Enjoy your walk! Don't forget sunscreen."
29+
},
30+
{
31+
"op": "append_event",
32+
"author": "user",
33+
"text": "What about tomorrow?"
34+
},
35+
{
36+
"op": "append_event",
37+
"author": "assistant",
38+
"text": "Tomorrow will be cloudy with a chance of rain."
39+
},
40+
{
41+
"op": "read_back",
42+
"expected_event_count": 6
43+
}
44+
]
45+
}

‎tests/sessions/replay_cases/03_tool_call.json‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
{
2+
"case_id": "03_tool_call",
3+
"description": "Tool call conversation: user query triggers a function_call, followed by function_response and final answer",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-003"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "Search for Python async tutorials"
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"function_call": {
19+
"name": "web_search",
20+
"args": {
21+
"query": "Python async tutorials"
22+
}
23+
}
24+
},
25+
{
26+
"op": "append_event",
27+
"author": "user",
28+
"function_response": {
29+
"name": "web_search",
30+
"response": {
31+
"results": [
32+
"AsyncIO Tutorial for Beginners",
33+
"Advanced Async Patterns in Python"
34+
]
35+
}
36+
}
37+
},
38+
{
39+
"op": "append_event",
40+
"author": "assistant",
41+
"text": "I found two relevant tutorials: AsyncIO Tutorial for Beginners and Advanced Async Patterns in Python."
42+
},
43+
{
44+
"op": "read_back",
45+
"expected_event_count": 4
46+
}
47+
]
48+
}

‎tests/sessions/replay_cases/04_state_update.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"case_id": "04_state_update",
3+
"description": "State update: multiple writes and overwrites to session state via event state_delta",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-004",
8+
"state": {
9+
"counter": 0,
10+
"status": "idle"
11+
}
12+
},
13+
"operations": [
14+
{
15+
"op": "append_event",
16+
"author": "user",
17+
"text": "Increment counter",
18+
"state_delta": {
19+
"counter": 1
20+
}
21+
},
22+
{
23+
"op": "append_event",
24+
"author": "assistant",
25+
"text": "Counter is now 1"
26+
},
27+
{
28+
"op": "append_event",
29+
"author": "user",
30+
"text": "Update status and counter",
31+
"state_delta": {
32+
"counter": 5,
33+
"status": "active",
34+
"last_action": "bulk_update"
35+
}
36+
},
37+
{
38+
"op": "append_event",
39+
"author": "assistant",
40+
"text": "Updated to counter=5, status=active"
41+
},
42+
{
43+
"op": "read_back",
44+
"expected_state": {
45+
"counter": 5,
46+
"status": "active",
47+
"last_action": "bulk_update"
48+
},
49+
"expected_event_count": 4
50+
}
51+
]
52+
}

‎tests/sessions/replay_cases/05_memory_write_read.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"case_id": "05_memory_write_read",
3+
"description": "Memory write and read: store session content in memory and search for stored facts",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-005"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "My name is Alice and my favorite food is pizza."
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "Nice to meet you, Alice! Pizza is a great choice."
19+
},
20+
{
21+
"op": "append_event",
22+
"author": "user",
23+
"text": "I also enjoy hiking on weekends."
24+
},
25+
{
26+
"op": "append_event",
27+
"author": "assistant",
28+
"text": "Hiking is wonderful exercise. Where do you usually hike?"
29+
},
30+
{
31+
"op": "store_memory"
32+
},
33+
{
34+
"op": "search_memory",
35+
"query": "What food does Alice like?"
36+
},
37+
{
38+
"op": "read_back"
39+
}
40+
]
41+
}

‎tests/sessions/replay_cases/06_summary_generation.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"case_id": "06_summary_generation",
3+
"description": "Summary generation and retrieval: inject a session summary and verify its content and metadata",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-006"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "Let's discuss the new project."
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "Sure, what kind of project are you planning?"
19+
},
20+
{
21+
"op": "append_event",
22+
"author": "user",
23+
"text": "A web application for task management with real-time collaboration."
24+
},
25+
{
26+
"op": "append_event",
27+
"author": "assistant",
28+
"text": "That sounds ambitious. We should use React for the frontend and WebSocket for real-time features."
29+
},
30+
{
31+
"op": "append_event",
32+
"author": "user",
33+
"text": "Agreed. Let's also add file sharing capabilities."
34+
},
35+
{
36+
"op": "append_event",
37+
"author": "assistant",
38+
"text": "Good idea. We can integrate cloud storage APIs for that."
39+
},
40+
{
41+
"op": "inject_summary",
42+
"session_id": "s-006",
43+
"summary_text": "The user and assistant are planning a task management web application. Key decisions: use React frontend, WebSocket for real-time collaboration, and integrate cloud storage APIs for file sharing.",
44+
"original_event_count": 6,
45+
"compressed_event_count": 2
46+
},
47+
{
48+
"op": "read_back",
49+
"expected_event_count": 6
50+
}
51+
]
52+
}

‎tests/sessions/replay_cases/07_summary_truncation.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"case_id": "07_summary_truncation",
3+
"description": "Summary with event truncation: inject a summary to compress old events, verify summary event is present alongside remaining recent events",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-007"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "message 1 - beginning of long conversation"
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "response 1"
19+
},
20+
{
21+
"op": "append_event",
22+
"author": "user",
23+
"text": "message 2 - more context"
24+
},
25+
{
26+
"op": "append_event",
27+
"author": "assistant",
28+
"text": "response 2"
29+
},
30+
{
31+
"op": "append_event",
32+
"author": "user",
33+
"text": "message 3 - still more"
34+
},
35+
{
36+
"op": "append_event",
37+
"author": "assistant",
38+
"text": "response 3"
39+
},
40+
{
41+
"op": "append_event",
42+
"author": "user",
43+
"text": "message 4 - getting closer"
44+
},
45+
{
46+
"op": "append_event",
47+
"author": "assistant",
48+
"text": "response 4"
49+
},
50+
{
51+
"op": "append_event",
52+
"author": "user",
53+
"text": "message 5 - recent question about the plan"
54+
},
55+
{
56+
"op": "append_event",
57+
"author": "assistant",
58+
"text": "response 5 - answering the recent question"
59+
},
60+
{
61+
"op": "inject_summary",
62+
"session_id": "s-007",
63+
"summary_text": "Earlier in the conversation, the user and assistant discussed various topics including messages 1 through 4.",
64+
"original_event_count": 10,
65+
"compressed_event_count": 3
66+
},
67+
{
68+
"op": "read_back",
69+
"expected_event_count": 10
70+
}
71+
]
72+
}

‎tests/sessions/replay_cases/08_error_recovery.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"case_id": "08_error_recovery",
3+
"description": "Error recovery: duplicate event append must not produce duplicated events in the session",
4+
"session_setup": {
5+
"app_name": "replay_test",
6+
"user_id": "test_user",
7+
"session_id": "s-008"
8+
},
9+
"operations": [
10+
{
11+
"op": "append_event",
12+
"author": "user",
13+
"text": "Normal message"
14+
},
15+
{
16+
"op": "append_event",
17+
"author": "assistant",
18+
"text": "Normal response"
19+
},
20+
{
21+
"op": "duplicate_append"
22+
},
23+
{
24+
"op": "read_back",
25+
"expected_event_count": 2
26+
}
27+
]
28+
}

0 commit comments

Comments
 (0)