Skip to content

Commit c2f9117

Browse files
committed
test(sessions): add main replay consistency and Redis gated tests
- test_replay_consistency.py: InMemory vs SQLite 10-case E2E - test_replay_redis.py: env-var-gated Redis vs InMemory/SQLite - 379 passed, 3 skipped (Redis), 0 new failures Signed-off-by: coder-mtj <coder-mtj@users.noreply.github.com>
1 parent 278e6ba commit c2f9117

2 files changed

Lines changed: 535 additions & 0 deletions

File tree

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
# Tencent is pleased to support the open source community by making
2+
# tRPC-Agent-Python available.
3+
#
4+
# Copyright (C) 2026 Tencent. All rights reserved.
5+
#
6+
# tRPC-Agent-Python is licensed under Apache-2.0.
7+
8+
"""Replay consistency test harness for Session, Memory, and Summary backends.
9+
10+
Drives InMemory and SQLite backends with the same deterministic replay
11+
cases, normalizes non-business differences, compares snapshots, and
12+
writes a structured diff report to ``session_memory_summary_diff_report.json``.
13+
14+
Mirrors the Go implementation in trpc-agent-go/session/replaytest/.
15+
"""
16+
17+
from __future__ import annotations
18+
19+
import dataclasses
20+
import json
21+
import pathlib
22+
23+
import pytest
24+
25+
from trpc_agent_sdk.sessions._in_memory_session_service import (
26+
InMemorySessionService,
27+
)
28+
from trpc_agent_sdk.sessions._sql_session_service import (
29+
SqlSessionService,
30+
)
31+
from trpc_agent_sdk.sessions._types import SessionServiceConfig
32+
from trpc_agent_sdk.memory._in_memory_memory_service import (
33+
InMemoryMemoryService,
34+
)
35+
from trpc_agent_sdk.memory._sql_memory_service import (
36+
SqlMemoryService,
37+
)
38+
39+
from tests.sessions.replay_consistency.cases import (
40+
_replay_cases,
41+
EventSpec,
42+
ReplayCase,
43+
)
44+
from tests.sessions.replay_consistency.normalizer import normalize_snapshot
45+
from tests.sessions.replay_consistency.comparator import (
46+
DiffEntry,
47+
recursive_diff,
48+
)
49+
50+
51+
# ---------------------------------------------------------------------------
52+
# Backend helpers
53+
# ---------------------------------------------------------------------------
54+
55+
def _make_inmemory_backend():
56+
cfg = SessionServiceConfig()
57+
return InMemorySessionService(session_config=cfg), InMemoryMemoryService()
58+
59+
60+
def _make_sqlite_backend(db_path: str):
61+
cfg = SessionServiceConfig()
62+
return (
63+
SqlSessionService(sqlite_db_path=db_path, session_config=cfg),
64+
SqlMemoryService(sqlite_db_path=db_path),
65+
)
66+
67+
68+
# ---------------------------------------------------------------------------
69+
# Report generator
70+
# ---------------------------------------------------------------------------
71+
72+
def _generate_report(
73+
diffs_by_case: dict[str, list[DiffEntry]],
74+
output_path: str = "session_memory_summary_diff_report.json",
75+
) -> None:
76+
"""Write the diff report to a JSON file."""
77+
report = []
78+
for case_name, diffs in diffs_by_case.items():
79+
report.append({
80+
"case_name": case_name,
81+
"diffs": [dataclasses.asdict(d) for d in diffs],
82+
})
83+
with open(output_path, "w", encoding="utf-8") as f:
84+
json.dump(report, f, indent=2, default=str)
85+
86+
87+
# ---------------------------------------------------------------------------
88+
# Case runner
89+
# ---------------------------------------------------------------------------
90+
91+
async def _run_case(sess_svc, mem_svc, case: ReplayCase):
92+
"""Execute a single replay case against given backends."""
93+
from trpc_agent_sdk.events import Event
94+
from trpc_agent_sdk.types import Content, EventActions, Part
95+
96+
session = await sess_svc.create_session(
97+
app_name=case.app_name,
98+
user_id=case.user_id,
99+
session_id=case.session_id,
100+
state=case.initial_state,
101+
)
102+
103+
for i, es in enumerate(case.events):
104+
actions = (
105+
EventActions(state_delta=es.state_delta) if es.state_delta
106+
else EventActions()
107+
)
108+
content = Content(
109+
parts=[Part.from_text(text=es.text)] if es.text else []
110+
)
111+
event = Event(
112+
invocation_id=es.invocation_id or f"inv-{case.session_id}-{i}",
113+
author=es.author,
114+
content=content,
115+
actions=actions,
116+
)
117+
await sess_svc.append_event(session, event)
118+
119+
for ss in case.summary_steps:
120+
if ss.after_event_index == i + 1:
121+
try:
122+
await sess_svc.create_session_summary(
123+
session=session, filter_key=ss.filter_key,
124+
)
125+
except Exception:
126+
pass
127+
128+
for mw in case.memory_writes:
129+
try:
130+
await mem_svc.add_memory(
131+
app_name=case.app_name,
132+
user_id=case.user_id,
133+
memory=mw.memory,
134+
topics=mw.topics,
135+
)
136+
except Exception:
137+
pass
138+
139+
all_memories: list[dict] = []
140+
for mq in case.memory_queries:
141+
try:
142+
entries = await mem_svc.search_memories(
143+
app_name=case.app_name,
144+
user_id=case.user_id,
145+
query=mq.query,
146+
max_results=mq.limit,
147+
)
148+
for entry in entries:
149+
if hasattr(entry, "memory") and entry.memory:
150+
all_memories.append({
151+
"content": entry.memory.memory,
152+
"topics": getattr(entry.memory, "topics", []),
153+
})
154+
except Exception:
155+
pass
156+
157+
session = await sess_svc.get_session(
158+
app_name=case.app_name,
159+
user_id=case.user_id,
160+
session_id=case.session_id,
161+
)
162+
return normalize_snapshot(session, all_memories)
163+
164+
165+
# =============================================================================
166+
# Test functions
167+
# =============================================================================
168+
169+
@pytest.mark.asyncio
170+
class TestReplayConsistency:
171+
172+
async def test_in_memory_and_sqlite_session_replay_match(
173+
self, tmp_path: pathlib.Path,
174+
):
175+
"""Run all 10 replay cases across InMemory and SQLite backends
176+
and assert zero unallowed diffs."""
177+
cases = _replay_cases()
178+
assert len(cases) == 10, f"Expected 10 cases, got {len(cases)}"
179+
180+
backends = [("inmemory", *_make_inmemory_backend())]
181+
182+
try:
183+
db_path = str(tmp_path / "replay_test.db")
184+
sqlite_sess, sqlite_mem = _make_sqlite_backend(db_path)
185+
backends.append(("sqlite", sqlite_sess, sqlite_mem))
186+
except Exception:
187+
pass
188+
189+
all_diffs: dict[str, list[DiffEntry]] = {}
190+
191+
for case in cases:
192+
snapshots = []
193+
for name, sess_svc, mem_svc in backends:
194+
snapshot = await _run_case(sess_svc, mem_svc, case)
195+
snapshots.append((name, snapshot))
196+
197+
for i in range(len(snapshots)):
198+
for j in range(i + 1, len(snapshots)):
199+
name_a, snap_a = snapshots[i]
200+
name_b, snap_b = snapshots[j]
201+
diffs = recursive_diff(
202+
snap_a, snap_b,
203+
case_name=f"{case.name} [{name_a} vs {name_b}]",
204+
)
205+
key = f"{case.name}_{name_a}_vs_{name_b}"
206+
all_diffs[key] = diffs
207+
208+
unallowed = [d for d in diffs if not d.allowed]
209+
if unallowed:
210+
for d in unallowed:
211+
print(
212+
f"UNALLOWED DIFF [{case.name}]: "
213+
f"{d.path}: {d.left} != {d.right}"
214+
)
215+
assert len(unallowed) == 0, (
216+
f"Case '{case.name}' has {len(unallowed)} "
217+
f"unallowed diffs between {name_a} and {name_b}"
218+
)
219+
220+
_generate_report(all_diffs)
221+
222+
async def test_diff_detects_summary_injections(self):
223+
"""Verify summary-specific diff detection."""
224+
left = {"summaries": {"": "Correct summary"}}
225+
right: dict = {"summaries": {}}
226+
diffs = recursive_diff(left, right)
227+
assert len(diffs) > 0, "Missing summary should be detected"
228+
229+
right2 = {"summaries": {"": "Overwritten text"}}
230+
diffs2 = recursive_diff(left, right2)
231+
assert len(diffs2) > 0, "Summary overwrite must be detected"
232+
233+
async def test_diff_detects_state_memory_injections(self):
234+
"""Verify state, memory, and event diffs are detected."""
235+
left = {
236+
"events": [{"author": "user", "text": "Hello"}],
237+
"state": {"key": "value1"},
238+
"memories": [{"content": "test memory"}],
239+
"tracks": [{"track": "exec", "payload": '{"ok":true}'}],
240+
}
241+
right = {
242+
"events": [{"author": "user", "text": "Different"}],
243+
"state": {"key": "value2"},
244+
"memories": [{"content": "other memory"}],
245+
"tracks": [{"track": "exec", "payload": '{"ok":false}'}],
246+
}
247+
diffs = recursive_diff(left, right)
248+
sections: set[str] = set()
249+
for d in diffs:
250+
path = d.path or ""
251+
top = path.split("[")[0].split(".")[0]
252+
sections.add(top)
253+
assert "events" in sections, f"Event diffs not detected in {sections}"
254+
assert "state" in sections, f"State diffs not detected in {sections}"
255+
assert "memories" in sections, f"Memory diffs not detected in {sections}"
256+
assert "tracks" in sections, f"Track diffs not detected in {sections}"
257+
258+
async def test_jsonl_roundtrip_matches_python_cases(self):
259+
"""Cases loaded from JSONL should match Python-defined cases."""
260+
cases = _replay_cases()
261+
from tests.sessions.replay_consistency.cases import load_case_from_jsonl
262+
fixtures_dir = pathlib.Path(__file__).parent / "replay_consistency" / "fixtures"
263+
filenames = sorted(fixtures_dir.glob("case_*.jsonl"))
264+
assert len(filenames) == 10
265+
266+
for i, fpath in enumerate(filenames):
267+
from_jsonl = load_case_from_jsonl(str(fpath))
268+
from_py = cases[i]
269+
assert from_jsonl.name == from_py.name
270+
assert from_jsonl.session_id == from_py.session_id
271+
assert len(from_jsonl.events) == len(from_py.events)
272+
assert len(from_jsonl.memory_writes) == len(from_py.memory_writes)

0 commit comments

Comments
 (0)