Skip to content

Commit 44bb8df

Browse files
shsaihdsaiudhshsaihdsaiudh
committed
sessions: add multi-backend replay consistency coverage
Add deterministic Session, Memory, and Summary replay cases for in-memory, SQLite, and optional Redis backends. Fix summary ordering across SQL round trips and decode Redis state hashes without losing JSON value types.\n\nFixes #89\n\nRELEASE NOTES: NONE
1 parent e113610 commit 44bb8df

9 files changed

Lines changed: 660 additions & 7 deletions

‎docs/replay_consistency_design.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Session / Memory / Summary Replay Consistency
2+
3+
框架以 JSONL 记录标准操作，用固定 id 和顺序驱动后端。默认比较内存与 SQLite；环境变量可切轻量模式或加入 Redis。时间统一为占位符，memory 结果排序；事件、state 和工具数据仍严格比较。`allowed_diff` 必须声明原因。
4+
5+
Summary 同时比较文本及存储关系。summary event 携带 session id、递增 version 与 supersedes id，摘要丢失、旧版覆盖和归属错误不会被掩盖。active 与 historical events 分开校验。报告给出 session、event index 或 summary id、路径及两端值；测试还会对 10 条轨迹逐一注入差异。

‎session_memory_summary_diff_report.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"schema_version": 1,
3+
"reference_backend": "in_memory",
4+
"compared_backends": [
5+
"sqlite",
6+
"redis"
7+
],
8+
"generated_by": "tests/sessions/test_replay_consistency.py",
9+
"allowed_diff_source": "tests/sessions/replay_cases/session_memory_summary.jsonl",
10+
"duration_seconds": 1.7,
11+
"compared_sections": [
12+
"events",
13+
"state",
14+
"memory",
15+
"summary"
16+
],
17+
"normalization": [
18+
"timestamp precision",
19+
"memory result ordering"
20+
],
21+
"cases": [
22+
{"case_id":"single_turn","session_id":"replay-single_turn","status":"match","differences":[]},
23+
{"case_id":"multi_turn","session_id":"replay-multi_turn","status":"match","differences":[]},
24+
{"case_id":"tool_round_trip","session_id":"replay-tool_round_trip","status":"match","differences":[]},
25+
{"case_id":"state_overwrite","session_id":"replay-state_overwrite","status":"match","differences":[]},
26+
{"case_id":"memory_preference","session_id":"replay-memory_preference","status":"match","differences":[]},
27+
{"case_id":"memory_fact_update","session_id":"replay-memory_fact_update","status":"match","differences":[]},
28+
{"case_id":"summary_create","session_id":"replay-summary_create","status":"match","differences":[]},
29+
{"case_id":"summary_update","session_id":"replay-summary_update","status":"match","differences":[]},
30+
{"case_id":"summary_truncation","session_id":"replay-summary_truncation","status":"match","differences":[]},
31+
{"case_id":"retry_recovery","session_id":"replay-retry_recovery","status":"match","differences":[]}
32+
]
33+
}

‎tests/sessions/replay_cases/session_memory_summary.jsonl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{"case_id":"single_turn","description":"single user and assistant turn","operations":[{"type":"event","author":"user","role":"user","text":"Hello agent"},{"type":"event","author":"assistant","role":"model","text":"Hello user"}],"allowed_diff":[{"path":"events[*].timestamp","reason":"Backend clocks and SQL timestamp precision differ; event order and payload remain exact."}]}
2+
{"case_id":"multi_turn","description":"three consecutive conversation turns","operations":[{"type":"event","author":"user","role":"user","text":"My name is Lin"},{"type":"event","author":"assistant","role":"model","text":"Nice to meet you, Lin"},{"type":"event","author":"user","role":"user","text":"What is my name?"},{"type":"event","author":"assistant","role":"model","text":"Your name is Lin"},{"type":"event","author":"user","role":"user","text":"Thanks"},{"type":"event","author":"assistant","role":"model","text":"You are welcome"}],"allowed_diff":[{"path":"events[*].timestamp","reason":"Backend clocks and SQL timestamp precision differ; event order and payload remain exact."}]}
3+
{"case_id":"tool_round_trip","description":"function call and response survive persistence","operations":[{"type":"event","author":"user","role":"user","text":"Weather in Shenzhen?"},{"type":"function_call","author":"assistant","role":"model","name":"weather","args":{"city":"Shenzhen"}},{"type":"function_response","author":"tool","role":"user","name":"weather","response":{"temperature":31,"unit":"C"}},{"type":"event","author":"assistant","role":"model","text":"It is 31 C in Shenzhen"}],"allowed_diff":[{"path":"events[*].timestamp","reason":"Backend clocks and SQL timestamp precision differ; tool names and values remain exact."}]}
4+
{"case_id":"state_overwrite","description":"session, user and app state updates and overwrites","operations":[{"type":"state","author":"assistant","text":"state one","state_delta":{"theme":"light","user:language":"zh","app:region":"cn"}},{"type":"state","author":"assistant","text":"state two","state_delta":{"theme":"dark","counter":1}},{"type":"state","author":"assistant","text":"state three","state_delta":{"counter":2,"user:language":"en"}}],"allowed_diff":[{"path":"events[*].timestamp","reason":"Backend clocks and SQL timestamp precision differ; final state is compared exactly."}]}
5+
{"case_id":"memory_preference","description":"write and retrieve a user preference","operations":[{"type":"event","author":"user","role":"user","text":"I prefer jasmine tea"},{"type":"event","author":"assistant","role":"model","text":"I will remember your tea preference"},{"type":"memory_store"},{"type":"memory_search","label":"preference","query":"jasmine"}],"allowed_diff":[{"path":"memory.*[*].timestamp","reason":"Memory services format timestamps from backend-specific precision; content and author remain exact."}]}
6+
{"case_id":"memory_fact_update","description":"memory rewrite is idempotent and searchable","operations":[{"type":"event","author":"user","role":"user","text":"Project Atlas launches Friday","event_id":"memory-fact"},{"type":"memory_store"},{"type":"memory_store"},{"type":"memory_search","label":"fact","query":"Atlas","limit":5}],"allowed_diff":[{"path":"memory.*[*].timestamp","reason":"Memory services format timestamps from backend-specific precision; duplicate count and payload remain exact."}]}
7+
{"case_id":"summary_create","description":"create and read a first summary","operations":[{"type":"event","author":"user","role":"user","text":"First detail"},{"type":"event","author":"assistant","role":"model","text":"First answer"},{"type":"event","author":"user","role":"user","text":"Recent question"},{"type":"summary","text":"The user discussed the first detail.","keep_recent":1}],"allowed_diff":[{"path":"summary.*.updated_at","reason":"Summary update timestamps use backend clocks; identity, ownership, version and content remain exact."},{"path":"historical_events[*].timestamp","reason":"Historical event timestamps have backend-specific storage precision."}]}
8+
{"case_id":"summary_update","description":"new summary supersedes the previous version","operations":[{"type":"event","author":"user","role":"user","text":"Old topic"},{"type":"event","author":"assistant","role":"model","text":"Old answer"},{"type":"summary","text":"Old topic was discussed.","keep_recent":1},{"type":"event","author":"user","role":"user","text":"New topic"},{"type":"event","author":"assistant","role":"model","text":"New answer"},{"type":"summary","text":"Old and new topics were discussed.","keep_recent":1}],"allowed_diff":[{"path":"summary.*.updated_at","reason":"Summary update timestamps use backend clocks; overwrite links and versions remain exact."},{"path":"historical_events[*].timestamp","reason":"Historical event timestamps have backend-specific storage precision."}]}
9+
{"case_id":"summary_truncation","description":"summary plus retained and new events reconstruct context","operations":[{"type":"event","author":"user","role":"user","text":"Archived question one"},{"type":"event","author":"assistant","role":"model","text":"Archived answer one"},{"type":"event","author":"user","role":"user","text":"Retained question"},{"type":"summary","text":"One archived question was answered.","keep_recent":1},{"type":"event","author":"assistant","role":"model","text":"Answer after summary"}],"allowed_diff":[{"path":"summary.*.updated_at","reason":"Summary update timestamps use backend clocks; ownership and active position remain exact."},{"path":"historical_events[*].timestamp","reason":"Historical event timestamps have backend-specific storage precision."}]}
10+
{"case_id":"retry_recovery","description":"failed and duplicate writes do not create dirty state or events","operations":[{"type":"event","author":"user","role":"user","text":"Submit once","event_id":"retry-stable-id"},{"type":"retry_event","author":"user","role":"user","text":"Submit once","event_id":"retry-stable-id"},{"type":"fail_before_commit","state_delta":{"dirty":true}},{"type":"state","author":"assistant","text":"Committed","state_delta":{"status":"committed"}}],"allowed_diff":[{"path":"events[*].timestamp","reason":"Backend clocks and SQL timestamp precision differ; retry cardinality and committed state remain exact."}]}

0 commit comments

Comments
 (0)