From 9ccc46877aab856dd15833c46d1f4b9c9d13632e Mon Sep 17 00:00:00 2001 From: rello-code Date: Mon, 15 Jun 2026 12:27:28 -0400 Subject: [PATCH] feat(failure-modes): add 4 mem0 failure mode regression scenarios Adds deterministic add/update/query test fixtures covering the four failure modes from mem0ai/mem0#5235: - test_forgetting_stale_preference - test_polarity_update_not_merged - test_temporal_relevance_recent_over_old - test_superseded_memory_excluded_from_context Each scenario runs independently against a fresh Mem0 client and scores pass/fail per failure dimension. Pattern mirrors the existing scenario-runner style used across this benchmark suite. --- .../test_memory_scenarios.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 benchmarks/mem0_failure_modes/test_memory_scenarios.py diff --git a/benchmarks/mem0_failure_modes/test_memory_scenarios.py b/benchmarks/mem0_failure_modes/test_memory_scenarios.py new file mode 100644 index 0000000..e0f97a8 --- /dev/null +++ b/benchmarks/mem0_failure_modes/test_memory_scenarios.py @@ -0,0 +1,135 @@ +""" +Mem0 Failure Mode Scenarios +=========================== + +Deterministic add/update/query fixtures covering the four memory failure +modes identified in mem0ai/mem0#5235. Each scenario runs independently +against a fresh mem0 client and scores pass/fail on the failure dimension. + +Failure modes covered: + 1. Forgetting - stale preferences deprioritised after update + 2. Overwrite - polarity-reversing update strictly supersedes old fact + 3. Temporal - recent fact wins over older conflicting fact + 4. Conflict - newer resolution wins over older conflicting resolution + +Usage: + pytest benchmarks/mem0_failure_modes/test_memory_scenarios.py -v +""" +from __future__ import annotations + +import os +import pytest +from mem0 import Memory + +USER_ID = "bench_user_failure_modes" + + +@pytest.fixture(scope="module") +def mem(): + """Return a fresh in-memory Mem0 client for the test session.""" + config = { + "vector_store": { + "provider": "qdrant", + "config": {"host": "localhost", "port": 6333}, + } + } + client = Memory.from_config(config) + yield client + # Cleanup after module + client.delete_all(user_id=USER_ID) + + +# --------------------------------------------------------------------------- +# 1. Forgetting: stale preference should be deprioritised after update +# --------------------------------------------------------------------------- + +def test_forgetting_stale_preference(mem): + """After updating a preference, the stale version must not surface first.""" + uid = f"{USER_ID}_forget" + mem.add("I prefer dark mode.", user_id=uid) + mem.add("I switched to light mode and love it.", user_id=uid) + + results = mem.search("What display mode do I prefer?", user_id=uid, limit=3) + top = results["results"][0]["memory"].lower() + + assert "light" in top, ( + f"Expected 'light mode' to rank first, got: {top!r}" + ) + mem.delete_all(user_id=uid) + + +# --------------------------------------------------------------------------- +# 2. Overwrite: polarity-reversing update must strictly replace old fact +# --------------------------------------------------------------------------- + +def test_polarity_update_not_merged(mem): + """ + Opposite-polarity update (notifications enabled -> disabled) must + produce exactly one memory with the new polarity; the old one must + be absent from the top result. + """ + uid = f"{USER_ID}_polarity" + mem.add("I have notifications enabled.", user_id=uid) + mem.add("I turned notifications off.", user_id=uid) + + results = mem.search("Are my notifications on or off?", user_id=uid, limit=3) + memories_text = " ".join( + r["memory"].lower() for r in results["results"] + ) + + assert "off" in memories_text or "disabled" in memories_text, ( + f"Expected 'off/disabled' in results, got: {memories_text!r}" + ) + assert "enabled" not in results["results"][0]["memory"].lower(), ( + "Stale 'enabled' fact must not rank first after polarity update." + ) + mem.delete_all(user_id=uid) + + +# --------------------------------------------------------------------------- +# 3. Temporal relevance: recent fact wins over older conflicting fact +# --------------------------------------------------------------------------- + +def test_temporal_relevance_recent_over_old(mem): + """The most recently added conflicting fact must rank above older ones.""" + uid = f"{USER_ID}_temporal" + mem.add("My favourite cuisine is Italian.", user_id=uid) + mem.add("My favourite cuisine is Japanese, actually.", user_id=uid) + mem.add("Actually I love Thai food the most now.", user_id=uid) + + results = mem.search("What is my favourite cuisine?", user_id=uid, limit=3) + top = results["results"][0]["memory"].lower() + + assert "thai" in top, ( + f"Expected 'Thai' to rank first (most recent), got: {top!r}" + ) + mem.delete_all(user_id=uid) + + +# --------------------------------------------------------------------------- +# 4. Conflict resolution: newer fact wins over older conflicting resolution +# --------------------------------------------------------------------------- + +def test_superseded_memory_excluded_from_context(mem): + """ + A superseded memory must not appear in retrieval context; only the + resolution memory should be returned for the conflicting topic. + """ + uid = f"{USER_ID}_conflict" + mem.add("I use Python for all my scripts.", user_id=uid) + mem.add("I now use TypeScript for all my scripts, not Python.", user_id=uid) + + results = mem.search("What language do I use for scripts?", user_id=uid, limit=5) + memories = [r["memory"].lower() for r in results["results"]] + + # The superseded Python-only memory must not be the top result + assert "typescript" in memories[0], ( + f"Expected TypeScript resolution to rank first, got: {memories[0]!r}" + ) + # Ideally the old Python-only memory is absent or ranked lower + for i, m in enumerate(memories): + if "python" in m and "typescript" not in m and i == 0: + pytest.fail( + f"Superseded Python-only memory ranked first: {m!r}" + ) + mem.delete_all(user_id=uid)