|
| 1 | +# Tencent is pleased to support the open source community by making |
| 2 | +# tRPC-Agent-Python available. |
| 3 | +# |
| 4 | +# Copyright (C) 2026 Tencent. All rights reserved. |
| 5 | +# |
| 6 | +# tRPC-Agent-Python is licensed under Apache-2.0. |
| 7 | + |
| 8 | +"""Replay consistency test harness for Session, Memory, and Summary backends. |
| 9 | +
|
| 10 | +Drives InMemory and SQLite backends with the same deterministic replay |
| 11 | +cases, normalizes non-business differences, compares snapshots, and |
| 12 | +writes a structured diff report to ``session_memory_summary_diff_report.json``. |
| 13 | +
|
| 14 | +Mirrors the Go implementation in trpc-agent-go/session/replaytest/. |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import dataclasses |
| 20 | +import json |
| 21 | +import pathlib |
| 22 | + |
| 23 | +import pytest |
| 24 | + |
| 25 | +from trpc_agent_sdk.sessions._in_memory_session_service import ( |
| 26 | + InMemorySessionService, |
| 27 | +) |
| 28 | +from trpc_agent_sdk.sessions._sql_session_service import ( |
| 29 | + SqlSessionService, |
| 30 | +) |
| 31 | +from trpc_agent_sdk.sessions._types import SessionServiceConfig |
| 32 | +from trpc_agent_sdk.memory._in_memory_memory_service import ( |
| 33 | + InMemoryMemoryService, |
| 34 | +) |
| 35 | +from trpc_agent_sdk.memory._sql_memory_service import ( |
| 36 | + SqlMemoryService, |
| 37 | +) |
| 38 | + |
| 39 | +from tests.sessions.replay_consistency.cases import ( |
| 40 | + _replay_cases, |
| 41 | + EventSpec, |
| 42 | + ReplayCase, |
| 43 | +) |
| 44 | +from tests.sessions.replay_consistency.normalizer import normalize_snapshot |
| 45 | +from tests.sessions.replay_consistency.comparator import ( |
| 46 | + DiffEntry, |
| 47 | + recursive_diff, |
| 48 | +) |
| 49 | + |
| 50 | + |
| 51 | +# --------------------------------------------------------------------------- |
| 52 | +# Backend helpers |
| 53 | +# --------------------------------------------------------------------------- |
| 54 | + |
| 55 | +def _make_inmemory_backend(): |
| 56 | + cfg = SessionServiceConfig() |
| 57 | + return InMemorySessionService(session_config=cfg), InMemoryMemoryService() |
| 58 | + |
| 59 | + |
| 60 | +def _make_sqlite_backend(db_path: str): |
| 61 | + cfg = SessionServiceConfig() |
| 62 | + return ( |
| 63 | + SqlSessionService(sqlite_db_path=db_path, session_config=cfg), |
| 64 | + SqlMemoryService(sqlite_db_path=db_path), |
| 65 | + ) |
| 66 | + |
| 67 | + |
| 68 | +# --------------------------------------------------------------------------- |
| 69 | +# Report generator |
| 70 | +# --------------------------------------------------------------------------- |
| 71 | + |
| 72 | +def _generate_report( |
| 73 | + diffs_by_case: dict[str, list[DiffEntry]], |
| 74 | + output_path: str = "session_memory_summary_diff_report.json", |
| 75 | +) -> None: |
| 76 | + """Write the diff report to a JSON file.""" |
| 77 | + report = [] |
| 78 | + for case_name, diffs in diffs_by_case.items(): |
| 79 | + report.append({ |
| 80 | + "case_name": case_name, |
| 81 | + "diffs": [dataclasses.asdict(d) for d in diffs], |
| 82 | + }) |
| 83 | + with open(output_path, "w", encoding="utf-8") as f: |
| 84 | + json.dump(report, f, indent=2, default=str) |
| 85 | + |
| 86 | + |
| 87 | +# --------------------------------------------------------------------------- |
| 88 | +# Case runner |
| 89 | +# --------------------------------------------------------------------------- |
| 90 | + |
| 91 | +async def _run_case(sess_svc, mem_svc, case: ReplayCase): |
| 92 | + """Execute a single replay case against given backends.""" |
| 93 | + from trpc_agent_sdk.events import Event |
| 94 | + from trpc_agent_sdk.types import Content, EventActions, Part |
| 95 | + |
| 96 | + session = await sess_svc.create_session( |
| 97 | + app_name=case.app_name, |
| 98 | + user_id=case.user_id, |
| 99 | + session_id=case.session_id, |
| 100 | + state=case.initial_state, |
| 101 | + ) |
| 102 | + |
| 103 | + for i, es in enumerate(case.events): |
| 104 | + actions = ( |
| 105 | + EventActions(state_delta=es.state_delta) if es.state_delta |
| 106 | + else EventActions() |
| 107 | + ) |
| 108 | + content = Content( |
| 109 | + parts=[Part.from_text(text=es.text)] if es.text else [] |
| 110 | + ) |
| 111 | + event = Event( |
| 112 | + invocation_id=es.invocation_id or f"inv-{case.session_id}-{i}", |
| 113 | + author=es.author, |
| 114 | + content=content, |
| 115 | + actions=actions, |
| 116 | + ) |
| 117 | + await sess_svc.append_event(session, event) |
| 118 | + |
| 119 | + for ss in case.summary_steps: |
| 120 | + if ss.after_event_index == i + 1: |
| 121 | + try: |
| 122 | + await sess_svc.create_session_summary( |
| 123 | + session=session, filter_key=ss.filter_key, |
| 124 | + ) |
| 125 | + except Exception: |
| 126 | + pass |
| 127 | + |
| 128 | + for mw in case.memory_writes: |
| 129 | + try: |
| 130 | + await mem_svc.add_memory( |
| 131 | + app_name=case.app_name, |
| 132 | + user_id=case.user_id, |
| 133 | + memory=mw.memory, |
| 134 | + topics=mw.topics, |
| 135 | + ) |
| 136 | + except Exception: |
| 137 | + pass |
| 138 | + |
| 139 | + all_memories: list[dict] = [] |
| 140 | + for mq in case.memory_queries: |
| 141 | + try: |
| 142 | + entries = await mem_svc.search_memories( |
| 143 | + app_name=case.app_name, |
| 144 | + user_id=case.user_id, |
| 145 | + query=mq.query, |
| 146 | + max_results=mq.limit, |
| 147 | + ) |
| 148 | + for entry in entries: |
| 149 | + if hasattr(entry, "memory") and entry.memory: |
| 150 | + all_memories.append({ |
| 151 | + "content": entry.memory.memory, |
| 152 | + "topics": getattr(entry.memory, "topics", []), |
| 153 | + }) |
| 154 | + except Exception: |
| 155 | + pass |
| 156 | + |
| 157 | + session = await sess_svc.get_session( |
| 158 | + app_name=case.app_name, |
| 159 | + user_id=case.user_id, |
| 160 | + session_id=case.session_id, |
| 161 | + ) |
| 162 | + return normalize_snapshot(session, all_memories) |
| 163 | + |
| 164 | + |
| 165 | +# ============================================================================= |
| 166 | +# Test functions |
| 167 | +# ============================================================================= |
| 168 | + |
| 169 | +@pytest.mark.asyncio |
| 170 | +class TestReplayConsistency: |
| 171 | + |
| 172 | + async def test_in_memory_and_sqlite_session_replay_match( |
| 173 | + self, tmp_path: pathlib.Path, |
| 174 | + ): |
| 175 | + """Run all 10 replay cases across InMemory and SQLite backends |
| 176 | + and assert zero unallowed diffs.""" |
| 177 | + cases = _replay_cases() |
| 178 | + assert len(cases) == 10, f"Expected 10 cases, got {len(cases)}" |
| 179 | + |
| 180 | + backends = [("inmemory", *_make_inmemory_backend())] |
| 181 | + |
| 182 | + try: |
| 183 | + db_path = str(tmp_path / "replay_test.db") |
| 184 | + sqlite_sess, sqlite_mem = _make_sqlite_backend(db_path) |
| 185 | + backends.append(("sqlite", sqlite_sess, sqlite_mem)) |
| 186 | + except Exception: |
| 187 | + pass |
| 188 | + |
| 189 | + all_diffs: dict[str, list[DiffEntry]] = {} |
| 190 | + |
| 191 | + for case in cases: |
| 192 | + snapshots = [] |
| 193 | + for name, sess_svc, mem_svc in backends: |
| 194 | + snapshot = await _run_case(sess_svc, mem_svc, case) |
| 195 | + snapshots.append((name, snapshot)) |
| 196 | + |
| 197 | + for i in range(len(snapshots)): |
| 198 | + for j in range(i + 1, len(snapshots)): |
| 199 | + name_a, snap_a = snapshots[i] |
| 200 | + name_b, snap_b = snapshots[j] |
| 201 | + diffs = recursive_diff( |
| 202 | + snap_a, snap_b, |
| 203 | + case_name=f"{case.name} [{name_a} vs {name_b}]", |
| 204 | + ) |
| 205 | + key = f"{case.name}_{name_a}_vs_{name_b}" |
| 206 | + all_diffs[key] = diffs |
| 207 | + |
| 208 | + unallowed = [d for d in diffs if not d.allowed] |
| 209 | + if unallowed: |
| 210 | + for d in unallowed: |
| 211 | + print( |
| 212 | + f"UNALLOWED DIFF [{case.name}]: " |
| 213 | + f"{d.path}: {d.left} != {d.right}" |
| 214 | + ) |
| 215 | + assert len(unallowed) == 0, ( |
| 216 | + f"Case '{case.name}' has {len(unallowed)} " |
| 217 | + f"unallowed diffs between {name_a} and {name_b}" |
| 218 | + ) |
| 219 | + |
| 220 | + _generate_report(all_diffs) |
| 221 | + |
| 222 | + async def test_diff_detects_summary_injections(self): |
| 223 | + """Verify summary-specific diff detection.""" |
| 224 | + left = {"summaries": {"": "Correct summary"}} |
| 225 | + right: dict = {"summaries": {}} |
| 226 | + diffs = recursive_diff(left, right) |
| 227 | + assert len(diffs) > 0, "Missing summary should be detected" |
| 228 | + |
| 229 | + right2 = {"summaries": {"": "Overwritten text"}} |
| 230 | + diffs2 = recursive_diff(left, right2) |
| 231 | + assert len(diffs2) > 0, "Summary overwrite must be detected" |
| 232 | + |
| 233 | + async def test_diff_detects_state_memory_injections(self): |
| 234 | + """Verify state, memory, and event diffs are detected.""" |
| 235 | + left = { |
| 236 | + "events": [{"author": "user", "text": "Hello"}], |
| 237 | + "state": {"key": "value1"}, |
| 238 | + "memories": [{"content": "test memory"}], |
| 239 | + "tracks": [{"track": "exec", "payload": '{"ok":true}'}], |
| 240 | + } |
| 241 | + right = { |
| 242 | + "events": [{"author": "user", "text": "Different"}], |
| 243 | + "state": {"key": "value2"}, |
| 244 | + "memories": [{"content": "other memory"}], |
| 245 | + "tracks": [{"track": "exec", "payload": '{"ok":false}'}], |
| 246 | + } |
| 247 | + diffs = recursive_diff(left, right) |
| 248 | + sections: set[str] = set() |
| 249 | + for d in diffs: |
| 250 | + path = d.path or "" |
| 251 | + top = path.split("[")[0].split(".")[0] |
| 252 | + sections.add(top) |
| 253 | + assert "events" in sections, f"Event diffs not detected in {sections}" |
| 254 | + assert "state" in sections, f"State diffs not detected in {sections}" |
| 255 | + assert "memories" in sections, f"Memory diffs not detected in {sections}" |
| 256 | + assert "tracks" in sections, f"Track diffs not detected in {sections}" |
| 257 | + |
| 258 | + async def test_jsonl_roundtrip_matches_python_cases(self): |
| 259 | + """Cases loaded from JSONL should match Python-defined cases.""" |
| 260 | + cases = _replay_cases() |
| 261 | + from tests.sessions.replay_consistency.cases import load_case_from_jsonl |
| 262 | + fixtures_dir = pathlib.Path(__file__).parent / "replay_consistency" / "fixtures" |
| 263 | + filenames = sorted(fixtures_dir.glob("case_*.jsonl")) |
| 264 | + assert len(filenames) == 10 |
| 265 | + |
| 266 | + for i, fpath in enumerate(filenames): |
| 267 | + from_jsonl = load_case_from_jsonl(str(fpath)) |
| 268 | + from_py = cases[i] |
| 269 | + assert from_jsonl.name == from_py.name |
| 270 | + assert from_jsonl.session_id == from_py.session_id |
| 271 | + assert len(from_jsonl.events) == len(from_py.events) |
| 272 | + assert len(from_jsonl.memory_writes) == len(from_py.memory_writes) |
0 commit comments