feat(onboard): opt-in [BETA] toggle for memory search + fix PR #2 review

Written by Minimax-M2.7 in Claude Code Harness.

Integrates sergi-rz's FTS5 memory search (#2) behind a beta feature
flag. Default OFF; users opt in through the onboarding wizard or by
editing .agent/memory/.features.json directly. Addresses the three
codex review findings on that PR at the same time.

Beta feature framework
- onboard_features.py (new): read/write .agent/memory/.features.json
  with a simple {key: {enabled: bool, beta: bool}} schema.
- onboard.py: adds a final "Optional features" step to the wizard.
  Default answer is No — beta features stay off unless the user
  explicitly opts in. --yes (non-interactive) path writes the
  features file too, all off.
- memory_search.py gates search/rebuild behind feature_enabled().
  --status still works regardless so users can see the toggle.

Codex review fixes on PR #2
- [P1] needs_rebuild now compares the set of memory files currently
  on disk with the set already in the index; any previously-indexed
  file that no longer exists flags the index stale. Without this,
  deleted/renamed files kept appearing in results until some
  unrelated file bumped the index.
- [P2] search_grep now passes explicit .md/.jsonl target paths to
  grep instead of scanning the whole .agent/memory/ tree. Source
  files (archive.py, auto_dream.py, etc.) no longer pollute
  fallback search results.
- [P3] .gitignore now ignores .agent/memory/.index/ correctly.
  The previous rule was cancelled by the !.agent/memory/**
  negation placed below it; moved the .index/ exclusion AFTER
  the negation so the later rule wins. Verified with
  git status --ignored.

Author: codejunkie99

.agent/memory/memory_search.py

@@ -1,17 +1,23 @@
 #!/usr/bin/env python3
 """
-Memory Search — SQLite FTS5 full-text search over .agent/memory/ files.
+Memory Search [BETA] — SQLite FTS5 full-text search over .agent/memory/ files.
 
 Indexes all .md and .jsonl files under .agent/memory/ and provides ranked
-keyword search. Falls back to grep if FTS5 is not available.
+keyword search. Falls back to grep (restricted to .md/.jsonl) if FTS5 is
+not available.
+
+BETA + opt-in: disabled by default. Enable via onboarding
+(agentic-stack <harness> --reconfigure) or by setting
+    {"memory_search_fts": {"enabled": true}}
+in .agent/memory/.features.json.
 
 Usage:
   python3 memory_search.py <query>       Search memories by keyword
   python3 memory_search.py --rebuild     Force rebuild the index
   python3 memory_search.py --status      Show index status
 
 The index is stored at .agent/memory/.index/memory.db and auto-rebuilds
-when any memory file changes. It is gitignored (derived data).
+when any memory file changes, is renamed, or is deleted.
 """
 import json
 import sys
@@ -22,6 +28,33 @@
 MEMORY_DIR = Path(__file__).resolve().parent
 INDEX_DIR = MEMORY_DIR / ".index"
 INDEX_PATH = INDEX_DIR / "memory.db"
+FEATURES_PATH = MEMORY_DIR / ".features.json"
+
+# Files we consider "memory documents" for both indexing and search.
+MEMORY_SUFFIXES = (".md", ".jsonl")
+
+
+def feature_enabled() -> bool:
+    """True iff `memory_search_fts` is opted in via .features.json.
+
+    Default OFF: beta features are explicit opt-in. Missing config file,
+    missing key, or `enabled: false` all resolve to disabled.
+    """
+    try:
+        data = json.loads(FEATURES_PATH.read_text(encoding="utf-8"))
+    except (FileNotFoundError, json.JSONDecodeError):
+        return False
+    entry = data.get("memory_search_fts") or {}
+    return bool(entry.get("enabled"))
+
+
+def _memory_files():
+    """Yield memory document paths, skipping the .index/ side directory."""
+    for f in MEMORY_DIR.rglob("*"):
+        if ".index" in f.parts:
+            continue
+        if f.suffix in MEMORY_SUFFIXES and f.is_file():
+            yield f
 
 
 def check_fts5() -> bool:
@@ -36,15 +69,36 @@ def check_fts5() -> bool:
 
 
 def needs_rebuild() -> bool:
-    """Check if index is stale (any source file newer than the index)."""
+    """True if the index is stale.
+
+    Stale means any of:
+      - index file does not exist
+      - a current memory file is newer than the index
+      - a file that WAS indexed no longer exists (delete / rename)
+
+    Without the third check, deleted files keep showing up in search
+    results until some unrelated edit bumps the index.
+    """
     if not INDEX_PATH.exists():
         return True
     index_mtime = INDEX_PATH.stat().st_mtime
-    for f in MEMORY_DIR.rglob("*"):
-        if ".index" in str(f):
-            continue
-        if f.suffix in (".md", ".jsonl") and f.stat().st_mtime > index_mtime:
+
+    current_rel = set()
+    for f in _memory_files():
+        if f.stat().st_mtime > index_mtime:
             return True
+        current_rel.add(str(f.relative_to(MEMORY_DIR)))
+
+    try:
+        conn = sqlite3.connect(INDEX_PATH)
+        indexed_rel = {row[0] for row in conn.execute("SELECT filename FROM memories")}
+        conn.close()
+    except sqlite3.OperationalError:
+        return True  # corrupt schema / unreadable — rebuild from scratch
+
+    # Any previously-indexed file no longer present? Rebuild to flush it.
+    if indexed_rel - current_rel:
+        return True
     return False
 
 
@@ -79,9 +133,7 @@ def build_index() -> int:
         USING fts5(filename, content, tokenize='porter unicode61')
     """)
     indexed = 0
-    for f in MEMORY_DIR.rglob("*"):
-        if ".index" in str(f):
-            continue
+    for f in _memory_files():
         try:
             if f.suffix == ".md":
                 content = f.read_text(encoding="utf-8")
@@ -90,7 +142,8 @@ def build_index() -> int:
             else:
                 continue
             rel_path = f.relative_to(MEMORY_DIR)
-            conn.execute("INSERT INTO memories VALUES (?, ?)", (str(rel_path), content))
+            conn.execute("INSERT INTO memories VALUES (?, ?)",
+                         (str(rel_path), content))
             indexed += 1
         except Exception:
             pass
@@ -99,7 +152,7 @@ def build_index() -> int:
     return indexed
 
 
-def search_fts5(query: str) -> list[tuple[str, str]]:
+def search_fts5(query: str):
     """Search the FTS5 index. Returns (filename, snippet) pairs."""
     if needs_rebuild():
         build_index()
@@ -123,19 +176,26 @@ def search_fts5(query: str) -> list[tuple[str, str]]:
     return rows
 
 
-def search_grep(query: str) -> list[tuple[str, str]]:
-    """Fallback search using grep when FTS5 is not available."""
+def search_grep(query: str):
+    """Fallback search using grep, restricted to memory document files.
+
+    Passing explicit target paths (not the whole directory) ensures we
+    don't match implementation files like archive.py or auto_dream.py —
+    keyword retrieval must only surface .md / .jsonl memory content.
+    """
+    targets = [str(f) for f in _memory_files()]
+    if not targets:
+        return []
     result = subprocess.run(
-        ["grep", "-ril", query, str(MEMORY_DIR)],
+        ["grep", "-ril", query, *targets],
         capture_output=True,
         text=True,
     )
-    files = [
-        f
-        for f in result.stdout.strip().split("\n")
-        if f and ".index" not in f
+    files = [f for f in result.stdout.strip().split("\n") if f]
+    return [
+        (Path(f).relative_to(MEMORY_DIR), f"(match in {Path(f).name})")
+        for f in files
     ]
-    return [(Path(f).relative_to(MEMORY_DIR), f"(match in {Path(f).name})") for f in files]
 
 
 def cmd_rebuild():
@@ -147,6 +207,13 @@ def cmd_rebuild():
 
 
 def cmd_status():
+    enabled = feature_enabled()
+    tag = "ENABLED" if enabled else "DISABLED (beta, opt-in)"
+    print(f"Feature: memory_search_fts [BETA] — {tag}")
+    if not enabled:
+        print("Enable via: agentic-stack <harness> --reconfigure")
+        print("Or edit .agent/memory/.features.json directly.")
+        return
     if not check_fts5():
         print("Mode: BASIC (grep fallback)")
         print("Reason: SQLite FTS5 not available in this Python build.")
@@ -163,24 +230,40 @@ def cmd_status():
     print(f"Location: {INDEX_PATH}")
 
 
+def _refuse_disabled():
+    print(
+        "memory_search [BETA] is disabled — opt-in only.\n"
+        "Enable via onboarding:  agentic-stack <harness> --reconfigure\n"
+        "Or set enabled=true for memory_search_fts in "
+        ".agent/memory/.features.json",
+        file=sys.stderr,
+    )
+    sys.exit(2)
+
+
 def main():
     args = sys.argv[1:]
 
     if not args or args[0] in ("-h", "--help"):
-        print("Usage:")
+        print("Usage [BETA, opt-in]:")
         print("  memory_search.py <query>     Search memories by keyword")
         print("  memory_search.py --rebuild   Force rebuild index")
         print("  memory_search.py --status    Show index status")
         sys.exit(0)
 
-    if args[0] == "--rebuild":
-        cmd_rebuild()
-        return
-
+    # --status always works (lets the user see whether the feature is on).
+    # All other commands require the opt-in flag.
     if args[0] == "--status":
         cmd_status()
         return
 
+    if not feature_enabled():
+        _refuse_disabled()
+
+    if args[0] == "--rebuild":
+        cmd_rebuild()
+        return
+
     query = " ".join(args)
     use_fts5 = check_fts5()
 

.gitignore

@@ -19,9 +19,11 @@ venv/
 .agent/memory/dream.log
 *.log
 
-# derived search index (auto-rebuilt)
-.agent/memory/.index/
-
 # do NOT ignore memory files — they are the point
 !.agent/memory/
 !.agent/memory/**
+
+# ...EXCEPT the derived search index (auto-rebuilt, per-machine).
+# Placed AFTER the negation above so the later rule wins.
+.agent/memory/.index/
+.agent/memory/.index/**

onboard.py

@@ -3,10 +3,11 @@
 import sys, os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from onboard_ui      import print_banner, intro, note, outro, step_done, BAR, R, MUTED, GREEN, PURPLE, WHITE, B
-from onboard_widgets import ask_text, ask_select, ask_confirm
-from onboard_render  import render
-from onboard_write   import write_prefs, is_customized, REL
+from onboard_ui       import print_banner, intro, note, outro, step_done, BAR, R, MUTED, GREEN, PURPLE, WHITE, B, ORANGE
+from onboard_widgets  import ask_text, ask_select, ask_confirm
+from onboard_render   import render
+from onboard_write    import write_prefs, is_customized, REL
+from onboard_features import write_features
 
 _CI_VARS = ("CI", "GITHUB_ACTIONS", "CIRCLECI", "BUILDKITE", "JENKINS_URL", "TRAVIS")
 
@@ -57,6 +58,16 @@ def _wizard(target, force):
                                  ["conventional commits", "free-form", "emoji"])
     a["review"]    = ask_select("Code review depth?",
                                  ["critical issues only", "everything"])
+
+    # ── Optional features (beta, opt-in) ───────────────────────────────
+    note("Optional features", [
+        f"{ORANGE}[BETA]{R}{MUTED} features — off by default, opt-in only.",
+        "You can change this later with  agentic-stack <harness> --reconfigure.",
+    ])
+    a["feature_memory_search"] = ask_confirm(
+        f"Enable FTS memory search  {ORANGE}[BETA]{R}?",
+        default=False,
+    )
     return a
 
 
@@ -71,19 +82,32 @@ def main():
 
     if yes:
         path = write_prefs(target, render({}), force=True)
+        # --yes defaults all optional beta features to off
+        features_file = write_features(target, {
+            "memory_search_fts": {"enabled": False, "beta": True},
+        })
         print(f"{GREEN}◆{R}  {WHITE}{B}PREFERENCES.md{R} written with defaults")
-        print(f"{MUTED}   {path}{R}\n")
+        print(f"{MUTED}   {path}{R}")
+        print(f"{MUTED}   {features_file} (all beta features off){R}\n")
         sys.exit(0)
 
     try:
         answers = _wizard(target, force=reconf)
         if answers is None:
             sys.exit(0)
         path = write_prefs(target, render(answers), force=reconf)
+        features = {
+            "memory_search_fts": {
+                "enabled": bool(answers.get("feature_memory_search")),
+                "beta": True,
+            },
+        }
+        features_file = write_features(target, features)
         outro([
             f"PREFERENCES.md written",
             f"{path}",
-            "Edit it any time — your AI re-reads it every session.",
+            f"Features: {features_file}",
+            "Edit either file any time — your AI re-reads them every session.",
             "Tip: git add .agent/memory/ to track your brain.",
         ])
     except KeyboardInterrupt:

onboard_features.py

@@ -0,0 +1,42 @@
+"""Optional feature toggles written by onboarding.
+
+Stored at `.agent/memory/.features.json` as a simple JSON map. Keeps the
+data separate from PREFERENCES.md so toggling a feature doesn't require
+regenerating the whole preferences file, and so downstream code can read
+it without parsing markdown.
+
+All features default to OFF. Users opt in explicitly during onboarding
+or by editing .features.json directly.
+"""
+import json
+import os
+
+FEATURES_REL = ".agent/memory/.features.json"
+
+
+def features_path(target_dir: str) -> str:
+    return os.path.join(target_dir, FEATURES_REL)
+
+
+def load_features(target_dir: str) -> dict:
+    try:
+        with open(features_path(target_dir), encoding="utf-8") as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def write_features(target_dir: str, features: dict) -> str:
+    path = features_path(target_dir)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(features, f, indent=2)
+        f.write("\n")
+    return path
+
+
+def is_enabled(target_dir: str, key: str) -> bool:
+    """True iff the feature is explicitly enabled. Off when file missing
+    or key absent — opt-in model."""
+    entry = load_features(target_dir).get(key) or {}
+    return bool(entry.get("enabled"))