Merge pull request #34 from huangrichao2020/codex/cjk-memory-search

fix(memory): recover short CJK search matches

Author: codejunkie99

.agent/memory/memory_search.py

@@ -21,6 +21,7 @@
 when any memory file changes, is renamed, or is deleted.
 """
 import json
+import re
 import shutil
 import sys
 import sqlite3
@@ -34,6 +35,7 @@
 
 # Files we consider "memory documents" for both indexing and search.
 MEMORY_SUFFIXES = (".md", ".jsonl")
+CJK_RE = re.compile(r"[\u3400-\u9fff]")
 
 
 def feature_enabled() -> bool:
@@ -174,6 +176,15 @@ def search_fts5(query: str):
             "SELECT filename, substr(content, 1, 300) FROM memories WHERE content LIKE ?",
             (f"%{query}%",),
         ).fetchall()
+
+    # SQLite's unicode61 tokenizer is good for mixed Latin/CJK text, but it
+    # does not segment every short CJK substring the way users expect. Keep the
+    # fast FTS path first, then recover exact CJK substring matches.
+    if not rows and CJK_RE.search(query):
+        rows = conn.execute(
+            "SELECT filename, substr(content, 1, 300) FROM memories WHERE content LIKE ?",
+            (f"%{query}%",),
+        ).fetchall()
     conn.close()
     return rows
 

test_memory_search.py

@@ -0,0 +1,81 @@
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent
+MEMORY_SEARCH = ROOT / ".agent" / "memory" / "memory_search.py"
+
+
+def load_memory_search():
+    spec = importlib.util.spec_from_file_location("memory_search", MEMORY_SEARCH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class MemorySearchTest(unittest.TestCase):
+    def with_memory_root(self):
+        tmp = tempfile.TemporaryDirectory()
+        root = Path(tmp.name) / ".agent" / "memory"
+        (root / "semantic").mkdir(parents=True)
+        (root / "episodic").mkdir()
+        (root / ".features.json").write_text(
+            json.dumps({"memory_search_fts": {"enabled": True}}),
+            encoding="utf-8",
+        )
+
+        module = load_memory_search()
+        module.MEMORY_DIR = root
+        module.INDEX_DIR = root / ".index"
+        module.INDEX_PATH = module.INDEX_DIR / "memory.db"
+        module.FEATURES_PATH = root / ".features.json"
+        return tmp, root, module
+
+    def test_short_cjk_query_matches_inside_longer_phrase(self):
+        tmp, root, memory_search = self.with_memory_root()
+        self.addCleanup(tmp.cleanup)
+
+        (root / "semantic" / "LESSONS.md").write_text(
+            "- 中文优先 when the user writes in Chinese.\n",
+            encoding="utf-8",
+        )
+
+        rows = memory_search.search_fts5("中文")
+
+        self.assertEqual(rows[0][0], "semantic/LESSONS.md")
+        self.assertIn("中文优先", rows[0][1])
+
+    def test_mixed_english_cjk_query_uses_fts5(self):
+        tmp, root, memory_search = self.with_memory_root()
+        self.addCleanup(tmp.cleanup)
+
+        (root / "semantic" / "LESSONS.md").write_text(
+            "- OpenClaw 飞书 document retrieval needs tab-aware recovery.\n",
+            encoding="utf-8",
+        )
+
+        rows = memory_search.search_fts5("OpenClaw 飞书")
+
+        self.assertEqual(rows[0][0], "semantic/LESSONS.md")
+        self.assertIn("OpenClaw", rows[0][1])
+        self.assertIn("飞书", rows[0][1])
+
+    def test_deleted_memory_file_is_removed_on_rebuild(self):
+        tmp, root, memory_search = self.with_memory_root()
+        self.addCleanup(tmp.cleanup)
+
+        note = root / "semantic" / "LESSONS.md"
+        note.write_text("- Keep stale anchors out of search results.\n", encoding="utf-8")
+        self.assertTrue(memory_search.search_fts5("stale anchors"))
+
+        note.unlink()
+        rows = memory_search.search_fts5("stale anchors")
+
+        self.assertEqual(rows, [])
+
+
+if __name__ == "__main__":
+    unittest.main()