fix(memory): close verification findings on async-merge PR (D-02/D-03/D-06/D-08/D-09)

D-06 (critical): consolidate_on_end was gated behind the LLMExtract early-return
in OnSessionEndWithProvenance, silently disabling consolidation whenever a user
set llm_extract=false (to disable episode extraction). These features are
conceptually independent. Fix: move the consolidation goroutine launch before the
LLMExtract guard, with its own independent gate (llm != nil, turns >= min,
ConsolidateOnEnd=true, LLMConsolidate=true). Verified: consolidation fires with
llm_extract=false; added regression test.

D-03: Consolidate had no upper-bound guard on LLM-returned entries; a
hallucinating LLM could expand the entry count. Fix: cap newEntries to
len(entries) before writeEntries.

D-02: writeEntries doc comment said "Caller must hold f.mu" but Consolidate
calls it holding factsDirLock instead of f.mu (safely, but undocumented). Fix:
clarify the locking contract — either f.mu (normal paths) or factsDirLock
(Consolidate) is sufficient; the two are never acquired together.

D-08: CHEATSHEET.md still documented the old judge behavior ("LLM judges →
merge or add"). Updated to reflect the new behavior (auto-add, deferred to
session-end consolidation).

D-09: CONFIG.md noted consolidate_on_end as the complement to merge-on-write,
but did not warn that disabling it causes near-duplicates to accumulate
permanently. Added an explicit note.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: jkyberneees

docs/CHEATSHEET.md

@@ -122,12 +122,13 @@ Settings: `model` (tiny/base/small/medium), `language` (ISO code, empty=auto), `
 ```
 RP.embed(newEntry) → cosine similarity vs each existing entry
 
-  cos > 0.7  ─────→ auto-merge (replace)
+  cos > 0.7  ─────→ simple merge (no LLM — substring or concatenation)
   cos < 0.3  ─────→ auto-add
-  0.3–0.7   ─────→ LLM judges → merge or add
+  0.3–0.7   ─────→ auto-add (deferred to session-end consolidation)
 ```
 
-Saves ~80% of LLM calls on memory writes.
+AddFact makes zero LLM calls. Near-duplicate dedup happens at session end
+via background consolidation (`consolidate_on_end`, default true).
 
 ### Memory Tool
 

docs/CONFIG.md

@@ -216,7 +216,7 @@ The `memory` section controls the persistent memory system (see [docs/MEMORY.md]
 | `buffer_lines` | 20 | Max turn summaries in session buffer |
 | `buffer_enabled` | true | Enable the turn-level buffer |
 | `merge_on_write` | true | Use go-vector RP similarity to auto-merge related entries (fast, no LLM — uses simple string merge) |
-| `consolidate_on_end` | true | At session end, run an LLM consolidation pass over `user.md` and `env.md` in a background goroutine. This is the quality complement to `merge_on_write`: merge-on-write handles obvious duplicates immediately (no LLM), while consolidation handles near-duplicates and paraphrases at session end with full LLM quality. Requires `llm_consolidate: true`. |
+| `consolidate_on_end` | true | At session end, run an LLM consolidation pass over `user.md` and `env.md` in a background goroutine. This is the quality complement to `merge_on_write`: merge-on-write handles obvious duplicates immediately (no LLM), while consolidation handles near-duplicates and paraphrases at session end with full LLM quality. Requires `llm_consolidate: true`. **Note:** facts in the borderline similarity band (0.3–0.7 cosine) are now always added immediately and only merged by this consolidation pass — if you set `consolidate_on_end: false`, near-duplicate facts will accumulate rather than being merged. |
 | `extract_on_end` | true | At session end (≥3 turns), extract a narrative episode summary via LLM for later recall |
 | `extract_facts` | **false** | **Opt-in.** At session end (≥3 turns), auto-extract a few **durable** facts (stable user preferences, project invariants) into `user.md`/`env.md`. Off by default — see the security note below. Independent of `extract_on_end`; to disable *all* end-of-session LLM extraction set `llm_extract: false`. |
 | `llm_search` | true | Use LLM to rerank candidates for **explicit** `memory search` calls (the `memory` tool). Per-turn recall (`FormatEpisodeContext`) always uses the cached go-vector index — no LLM call on the hot path regardless of this setting. |

internal/memory/async_merge_test.go

@@ -124,6 +124,36 @@ func TestConsolidateOnEnd_FiresAtSessionEnd(t *testing.T) {
 	t.Error("session-end consolidation did not reduce fact count within 3 seconds")
 }
 
+// TestConsolidateOnEnd_IndependentOfLLMExtract: consolidation must fire even
+// when llm_extract=false (episode extraction disabled). D-06 regression guard.
+func TestConsolidateOnEnd_IndependentOfLLMExtract(t *testing.T) {
+	dir := t.TempDir()
+	llm := &mockLLM{responses: map[string]string{
+		"memory entri": `["consolidated single fact"]`,
+	}}
+	cfg := DefaultMemoryConfig()
+	cfg.LLMExtract = boolPtr(false) // episodes off — must NOT suppress consolidation
+	cfg.ConsolidateOnEnd = boolPtr(true)
+	cfg.MergeOnWrite = boolPtr(false)
+	mm := NewMemoryManager(dir, llm, cfg)
+	_ = mm.AddFact("user", "prefers dark mode editors")
+	_ = mm.AddFact("user", "uses dark theme always")
+	entries0, _ := mm.facts.Entries("user")
+	if len(entries0) < 2 {
+		t.Fatalf("need 2 seeded entries, got %d", len(entries0))
+	}
+	msgs := []string{"user: hi", "assistant: ok", "user: more", "assistant: done"}
+	mm.OnSessionEndWithProvenance("sess-d06", 5, msgs, EpisodeProvenance{})
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if e, _ := mm.facts.Entries("user"); len(e) < 2 {
+			return // consolidation fired
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Error("consolidation should fire even with llm_extract=false")
+}
+
 // TestConsolidateOnEnd_FlagOff: with consolidate_on_end=false, fact count must
 // remain stable at session end (no consolidation LLM call).
 func TestConsolidateOnEnd_FlagOff(t *testing.T) {

internal/memory/facts.go

@@ -304,7 +304,13 @@ func (f *FactStore) Entries(target string) ([]string, error) {
 // writeEntries joins entries and writes them to disk atomically. It writes to a
 // UNIQUE temp file in the same directory and renames it into place, so two
 // FactStore instances writing the same directory concurrently can never clobber
-// a shared temp file. Caller must hold f.mu.
+// a shared temp file.
+//
+// Locking contract: callers must hold EITHER f.mu (normal mutation paths via
+// readModifyWrite) OR the process-wide factsDirLock for this directory
+// (Consolidate path in memory.go). These two locks are never acquired together,
+// so there is no deadlock risk. factsDirLock provides the same cross-instance
+// mutual exclusion that f.mu provides per-instance.
 func (f *FactStore) writeEntries(target string, entries []string) error {
 	content := strings.Join(entries, entrySep)
 	path := f.path(target)

internal/memory/memory.go

@@ -417,6 +417,11 @@ Entries for %s:
 	if len(newEntries) == 0 || (len(newEntries) == 1 && newEntries[0] == "") {
 		return nil // LLM returned nothing useful
 	}
+	// Guard against a hallucinating LLM expanding the entry count; the system
+	// prompt says "Never more than the original count" but that is advisory only.
+	if len(newEntries) > len(entries) {
+		newEntries = newEntries[:len(entries)]
+	}
 
 	// Security: scan LLM output before persisting
 	for _, entry := range newEntries {
@@ -514,7 +519,24 @@ func (m *MemoryManager) OnSessionEndWithProvenance(sessionID string, turns int,
 	if minTurns <= 0 {
 		minTurns = defaultMinTurnsForExtraction
 	}
-	// Shared preconditions for any end-of-session LLM extraction.
+
+	// Background consolidation is independent of episode/fact extraction —
+	// it fires based on its own gate so that llm_extract=false does not
+	// silently disable it (D-06). Requires an LLM client, llm_consolidate,
+	// and a minimum session length (same threshold reused for consistency).
+	if m.llm != nil && turns >= minTurns &&
+		m.cfg.ConsolidateOnEnd != nil && *m.cfg.ConsolidateOnEnd &&
+		m.cfg.LLMConsolidate != nil && *m.cfg.LLMConsolidate {
+		go func() {
+			for _, target := range []string{"user", "env"} {
+				// Best-effort: errors (e.g. only 1 entry, nothing to consolidate)
+				// are silently ignored — consolidation is a quality pass, not critical.
+				_ = m.Consolidate(target)
+			}
+		}()
+	}
+
+	// Preconditions shared by episode summary + fact extraction.
 	if m.cfg.LLMExtract == nil || !*m.cfg.LLMExtract || m.llm == nil || turns < minTurns || len(messages) == 0 {
 		return
 	}
@@ -531,20 +553,6 @@ func (m *MemoryManager) OnSessionEndWithProvenance(sessionID string, turns int,
 	if m.cfg.ExtractFacts != nil && *m.cfg.ExtractFacts && !prov.Untrusted {
 		m.extractFactsFromSession(convText)
 	}
-
-	// Background consolidation: recover the LLM-merge quality that AddFact no
-	// longer applies synchronously. Runs after fact extraction so any newly
-	// auto-extracted facts are also consolidated in the same pass.
-	if m.cfg.ConsolidateOnEnd != nil && *m.cfg.ConsolidateOnEnd &&
-		m.cfg.LLMConsolidate != nil && *m.cfg.LLMConsolidate && m.llm != nil {
-		go func() {
-			for _, target := range []string{"user", "env"} {
-				// Best-effort: errors (e.g. only 1 entry, nothing to consolidate)
-				// are silently ignored — consolidation is a quality pass, not critical.
-				_ = m.Consolidate(target)
-			}
-		}()
-	}
 }
 
 // buildConvText joins the session's message lines into a single transcript for