feat(failure-modes): add 4 mem0 failure mode regression scenarios

benchmarks/mem0_failure_modes/test_memory_scenarios.py

@@ -0,0 +1,135 @@
+"""
+Mem0 Failure Mode Scenarios
+===========================
+
+Deterministic add/update/query fixtures covering the four memory failure
+modes identified in mem0ai/mem0#5235. Each scenario runs independently
+against a fresh mem0 client and scores pass/fail on the failure dimension.
+
+Failure modes covered:
+  1. Forgetting       - stale preferences deprioritised after update
+  2. Overwrite        - polarity-reversing update strictly supersedes old fact
+  3. Temporal         - recent fact wins over older conflicting fact
+  4. Conflict         - newer resolution wins over older conflicting resolution
+
+Usage:
+    pytest benchmarks/mem0_failure_modes/test_memory_scenarios.py -v
+"""
+from __future__ import annotations
+
+import os
+import pytest
+from mem0 import Memory
+
+USER_ID = "bench_user_failure_modes"
+
+
+@pytest.fixture(scope="module")
+def mem():
+    """Return a fresh in-memory Mem0 client for the test session."""
+    config = {
+        "vector_store": {
+            "provider": "qdrant",
+            "config": {"host": "localhost", "port": 6333},
+        }
+    }
+    client = Memory.from_config(config)
+    yield client
+    # Cleanup after module
+    client.delete_all(user_id=USER_ID)
+
+
+# ---------------------------------------------------------------------------
+# 1. Forgetting: stale preference should be deprioritised after update
+# ---------------------------------------------------------------------------
+
+def test_forgetting_stale_preference(mem):
+    """After updating a preference, the stale version must not surface first."""
+    uid = f"{USER_ID}_forget"
+    mem.add("I prefer dark mode.", user_id=uid)
+    mem.add("I switched to light mode and love it.", user_id=uid)
+
+    results = mem.search("What display mode do I prefer?", user_id=uid, limit=3)
+    top = results["results"][0]["memory"].lower()
+
+    assert "light" in top, (
+        f"Expected 'light mode' to rank first, got: {top!r}"
+    )
+    mem.delete_all(user_id=uid)
+
+
+# ---------------------------------------------------------------------------
+# 2. Overwrite: polarity-reversing update must strictly replace old fact
+# ---------------------------------------------------------------------------
+
+def test_polarity_update_not_merged(mem):
+    """
+    Opposite-polarity update (notifications enabled -> disabled) must
+    produce exactly one memory with the new polarity; the old one must
+    be absent from the top result.
+    """
+    uid = f"{USER_ID}_polarity"
+    mem.add("I have notifications enabled.", user_id=uid)
+    mem.add("I turned notifications off.", user_id=uid)
+
+    results = mem.search("Are my notifications on or off?", user_id=uid, limit=3)
+    memories_text = " ".join(
+        r["memory"].lower() for r in results["results"]
+    )
+
+    assert "off" in memories_text or "disabled" in memories_text, (
+        f"Expected 'off/disabled' in results, got: {memories_text!r}"
+    )
+    assert "enabled" not in results["results"][0]["memory"].lower(), (
+        "Stale 'enabled' fact must not rank first after polarity update."
+    )
+    mem.delete_all(user_id=uid)
+
+
+# ---------------------------------------------------------------------------
+# 3. Temporal relevance: recent fact wins over older conflicting fact
+# ---------------------------------------------------------------------------
+
+def test_temporal_relevance_recent_over_old(mem):
+    """The most recently added conflicting fact must rank above older ones."""
+    uid = f"{USER_ID}_temporal"
+    mem.add("My favourite cuisine is Italian.", user_id=uid)
+    mem.add("My favourite cuisine is Japanese, actually.", user_id=uid)
+    mem.add("Actually I love Thai food the most now.", user_id=uid)
+
+    results = mem.search("What is my favourite cuisine?", user_id=uid, limit=3)
+    top = results["results"][0]["memory"].lower()
+
+    assert "thai" in top, (
+        f"Expected 'Thai' to rank first (most recent), got: {top!r}"
+    )
+    mem.delete_all(user_id=uid)
+
+
+# ---------------------------------------------------------------------------
+# 4. Conflict resolution: newer fact wins over older conflicting resolution
+# ---------------------------------------------------------------------------
+
+def test_superseded_memory_excluded_from_context(mem):
+    """
+    A superseded memory must not appear in retrieval context; only the
+    resolution memory should be returned for the conflicting topic.
+    """
+    uid = f"{USER_ID}_conflict"
+    mem.add("I use Python for all my scripts.", user_id=uid)
+    mem.add("I now use TypeScript for all my scripts, not Python.", user_id=uid)
+
+    results = mem.search("What language do I use for scripts?", user_id=uid, limit=5)
+    memories = [r["memory"].lower() for r in results["results"]]
+
+    # The superseded Python-only memory must not be the top result
+    assert "typescript" in memories[0], (
+        f"Expected TypeScript resolution to rank first, got: {memories[0]!r}"
+    )
+    # Ideally the old Python-only memory is absent or ranked lower
+    for i, m in enumerate(memories):
+        if "python" in m and "typescript" not in m and i == 0:
+            pytest.fail(
+                f"Superseded Python-only memory ranked first: {m!r}"
+            )
+    mem.delete_all(user_id=uid)