feat: Phase 6 — world-class retrieval (MAGMA + GAM inspired)

Based on two April 2026 research papers:
- MAGMA (arxiv:2601.03236): 0.700 on LoCoMo benchmark
- GAM (arxiv:2604.12285): 40.0 F1 on LoCoMo benchmark

Changes:
- Intent classifier: Why/When/Who/How/What detection (no LLM needed)
- Intent-aware graph traversal: edge weights boosted by query intent
  - Why queries → boost caused_by, led_to edges (5x)
  - When queries → boost learned_in, temporal edges (4x)
  - Who queries → boost touches, entity edges (5x)
  - How queries → boost part_of, depends_on edges (4x)
- Dual-stream ingestion: fast path (sync) + slow path (async goroutine)
  - Fast: store node + temporal backbone edge, return immediately
  - Slow: infer causal edges, link entities, consolidate
- Semantic boundary detection: cosine distance-based topic shift detection
  - Consolidate only at semantic boundaries (not arbitrary session ends)
  - Prevents transient noise contaminating long-term memory
- New CLI: yaad intent [query]
- New MCP tool: yaad_intent
- 17 tests passing (3 new Phase 6 tests)

Author: Patel230

PLAN.md

@@ -994,6 +994,63 @@ yaad/
 - [ ] Session replay
 - [ ] WebSocket/SSE streaming for real-time memory updates
 
+### Phase 6: World-Class Retrieval (MAGMA + GAM inspired)
+
+Based on two April 2026 papers:
+- **MAGMA** (arxiv:2601.03236) — Multi-Graph Agentic Memory Architecture, 0.700 on LoCoMo
+- **GAM** (arxiv:2604.12285) — Hierarchical Graph-based Agentic Memory, 40.0 F1 on LoCoMo
+
+#### Key Innovations
+
+**Intent-Aware Retrieval (MAGMA)**
+```
+Query: "Why did we choose NATS?"
+  → Intent: Why → boost causal edges (caused_by, led_to)
+  → Traverse: decision → convention → bug (causal chain)
+
+Query: "When did we fix the auth bug?"
+  → Intent: When → boost temporal edges (learned_in, session order)
+  → Traverse: temporal backbone
+
+Query: "What is the auth subsystem?"
+  → Intent: What → boost entity/spec edges
+  → Traverse: entity → spec → convention
+```
+
+**Dual-Stream Ingestion (MAGMA + GAM)**
+```
+Remember("Use jose for JWT")
+  │
+  ├── FAST PATH (sync, <1ms)
+  │   ├── Privacy filter
+  │   ├── Create node
+  │   ├── Add temporal edge (temporal backbone)
+  │   └── Return immediately ← agent not blocked
+  │
+  └── SLOW PATH (async goroutine)
+      ├── Extract entities (LLM or regex)
+      ├── Infer causal edges (LLM: "what caused this?")
+      ├── Link entity graph
+      └── Update semantic edges
+```
+
+**Semantic Boundary Detection (GAM)**
+```
+Session buffer fills with events
+  → LLM (or heuristic) detects topic shift
+  → Consolidate buffer → topic node
+  → Reset buffer
+  → Long-term memory only updated at semantic boundaries
+  → Prevents transient noise contaminating long-term memory
+```
+
+- [ ] Intent classifier (Why/When/Who/How/What) — regex + keyword, no LLM needed
+- [ ] Intent-aware edge weight boosting in graph traversal
+- [ ] Dual-stream ingestion (fast sync + slow async goroutine)
+- [ ] Semantic boundary detection (heuristic: cosine distance between consecutive summaries)
+- [ ] Topic consolidation at boundaries (not just at session end)
+- [ ] Multi-factor re-ranking: semantic × temporal × confidence × role
+
 ### Phase 5: Team & Scale
 - [ ] Team memory sharing (namespaced)
 - [ ] Skill/procedural memory (replayable workflows)

cmd/yaad/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/yaadmemory/yaad/internal/storage"
 	yaadsync "github.com/yaadmemory/yaad/internal/sync"
 	"github.com/yaadmemory/yaad/internal/tui"
+	intentpkg "github.com/yaadmemory/yaad/internal/intent"
 )
 
 var version = "0.1.0"
@@ -295,7 +296,7 @@ func init() {
 		hookCmd, setupCmd, replayCmd,
 		exportJSONCmd, exportMarkdownCmd, exportObsidianCmd, importJSONCmd,
 		skillStoreCmd, skillListCmd, skillReplayCmd, benchCmd,
-		syncCmd, tuiCmd)
+		syncCmd, tuiCmd, intentCmd)
 }
 
 func truncate(s string, n int) string {
@@ -675,6 +676,23 @@ var tuiCmd = &cobra.Command{
 	},
 }
 
+var intentCmd = &cobra.Command{
+	Use:   "intent [query]",
+	Short: "Classify query intent (Why/When/Who/How/What) for intent-aware retrieval",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		query := strings.Join(args, " ")
+		i := intentpkg.Classify(query)
+		w := intentpkg.Weights(i)
+		fmt.Printf("Query:  %s\n", query)
+		fmt.Printf("Intent: %s\n", i.String())
+		fmt.Printf("Edge weights:\n")
+		fmt.Printf("  caused_by:  %.1f  led_to:    %.1f\n", w.CausedBy, w.LedTo)
+		fmt.Printf("  learned_in: %.1f  touches:   %.1f\n", w.LearnedIn, w.Touches)
+		fmt.Printf("  part_of:    %.1f  relates_to: %.1f\n", w.PartOf, w.RelatesTo)
+	},
+}
+
 var syncCmd = &cobra.Command{
 	Use:   "sync",
 	Short: "Sync memories via git chunks (.yaad/chunks/*.jsonl.gz)",

integration_test.go

@@ -10,13 +10,17 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/yaadmemory/yaad/internal/agentconfig"
 	"github.com/yaadmemory/yaad/internal/bench"
+	"github.com/yaadmemory/yaad/internal/boundary"
 	"github.com/yaadmemory/yaad/internal/embeddings"
 	"github.com/yaadmemory/yaad/internal/engine"
 	"github.com/yaadmemory/yaad/internal/exportimport"
 	"github.com/yaadmemory/yaad/internal/hooks"
+	"github.com/yaadmemory/yaad/internal/ingest"
+	intentpkg "github.com/yaadmemory/yaad/internal/intent"
 	"github.com/yaadmemory/yaad/internal/server"
 	"github.com/yaadmemory/yaad/internal/skill"
 	"github.com/yaadmemory/yaad/internal/storage"
@@ -548,6 +552,129 @@ func TestGitSync(t *testing.T) {
 	}
 }
 
+func TestPhase6IntentClassifier(t *testing.T) {
+	cases := []struct {
+		query    string
+		expected intentpkg.Intent
+	}{
+		{"why did we choose NATS over Redis?", intentpkg.IntentWhy},
+		{"when did we fix the auth bug?", intentpkg.IntentWhen},
+		{"how to deploy the application?", intentpkg.IntentHow},
+		{"what is the auth subsystem?", intentpkg.IntentWhat},
+		{"which library should I use for JWT?", intentpkg.IntentWho},
+		{"recall auth middleware", intentpkg.IntentGeneral},
+	}
+	for _, c := range cases {
+		got := intentpkg.Classify(c.query)
+		if got != c.expected {
+			t.Errorf("Classify(%q) = %s, want %s", c.query, got, c.expected)
+		}
+	}
+}
+
+func TestPhase6IntentAwareRetrieval(t *testing.T) {
+	eng, cleanup := setup(t)
+	defer cleanup()
+
+	// Seed memories
+	decision, _ := eng.Remember(engine.RememberInput{Type: "decision", Content: "Chose NATS over Redis Streams for event bus", Scope: "project"})
+	convention, _ := eng.Remember(engine.RememberInput{Type: "convention", Content: "Use NATS client v2 for all event publishing", Scope: "project"})
+
+	// Link: decision led_to convention
+	eng.Graph().AddEdge(&storage.Edge{
+		ID: "e-test", FromID: decision.ID, ToID: convention.ID, Type: "led_to", Weight: 1.0,
+	})
+
+	// Why query should find the decision via causal traversal
+	result, err := eng.Recall(engine.RecallOpts{Query: "why NATS", Depth: 2, Limit: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Nodes) == 0 {
+		t.Error("intent-aware recall returned no nodes")
+	}
+	// Should find both decision and convention via causal chain
+	found := map[string]bool{}
+	for _, n := range result.Nodes {
+		found[n.Type] = true
+	}
+	t.Logf("Why query found types: %v", found)
+}
+
+func TestPhase6DualStream(t *testing.T) {
+	eng, cleanup := setup(t)
+	defer cleanup()
+
+	ds := ingest.New(eng)
+	defer ds.Stop()
+
+	// Fast path should return immediately
+	node, err := ds.Remember(engine.RememberInput{
+		Type: "convention", Content: "Use jose not jsonwebtoken", Scope: "project",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if node.ID == "" {
+		t.Error("dual stream: empty node ID")
+	}
+
+	// Second remember should create temporal backbone edge
+	node2, err := ds.Remember(engine.RememberInput{
+		Type: "decision", Content: "Chose RS256 for JWT", Scope: "project",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if node2.ID == "" {
+		t.Error("dual stream: second node empty ID")
+	}
+
+	// Give slow path time to run and release DB lock
+	time.Sleep(200 * time.Millisecond)
+
+	// Verify temporal backbone edge exists
+	edges, _ := eng.Store().GetEdgesFrom(node.ID)
+	hasTemporalEdge := false
+	for _, e := range edges {
+		if e.ToID == node2.ID && e.Type == "learned_in" {
+			hasTemporalEdge = true
+		}
+	}
+	if !hasTemporalEdge {
+		t.Error("dual stream: temporal backbone edge not created")
+	}
+}
+
+func TestPhase6BoundaryDetector(t *testing.T) {
+	// Test buffer overflow boundary (deterministic)
+	det := boundary.New(3, 0.99) // very high threshold, only overflow triggers
+	det.Add("item 1 about auth")
+	det.Add("item 2 about auth")
+	if !det.Add("item 3 about auth") {
+		t.Error("buffer overflow should trigger boundary")
+	}
+
+	// Test flush
+	det2 := boundary.New(10, 0.3)
+	det2.Add("content about authentication")
+	det2.Add("more about JWT tokens")
+	buf := det2.Flush()
+	if len(buf) != 2 {
+		t.Errorf("flush: expected 2 items, got %d", len(buf))
+	}
+	if det2.Size() != 0 {
+		t.Error("flush: buffer should be empty after flush")
+	}
+
+	// Test semantic distance detection (non-deterministic, just verify no panic)
+	det3 := boundary.New(20, 0.3)
+	det3.Add("Use jose for JWT authentication in Node.js")
+	det3.Add("PostgreSQL database connection pooling configuration")
+	// May or may not trigger — just verify it runs without error
+	t.Logf("Boundary detector size after 2 items: %d", det3.Size())
+}
+
 func TestRESTAPI(t *testing.T) {
 	eng, cleanup := setup(t)
 	defer cleanup()

internal/boundary/detector.go

@@ -0,0 +1,131 @@
+// Package boundary implements semantic boundary detection for memory consolidation.
+// Based on GAM (arxiv:2604.12285): consolidate only at semantic boundaries,
+// not arbitrary session ends, to prevent transient noise contaminating long-term memory.
+package boundary
+
+import (
+	"math"
+	"strings"
+	"unicode"
+)
+
+// Detector detects semantic topic shifts in a stream of memory content.
+type Detector struct {
+	buffer    []string // recent content items
+	maxBuffer int      // max items before forced consolidation
+	threshold float64  // cosine distance threshold for boundary detection
+}
+
+// New creates a boundary detector.
+// threshold: 0.0-1.0, higher = more sensitive to topic shifts (default 0.3)
+func New(maxBuffer int, threshold float64) *Detector {
+	if maxBuffer <= 0 {
+		maxBuffer = 20
+	}
+	if threshold <= 0 {
+		threshold = 0.3
+	}
+	return &Detector{maxBuffer: maxBuffer, threshold: threshold}
+}
+
+// Add adds content to the buffer and returns true if a semantic boundary is detected.
+// A boundary means: consolidate the current buffer into a topic node.
+func (d *Detector) Add(content string) bool {
+	if len(d.buffer) == 0 {
+		d.buffer = append(d.buffer, content)
+		return false
+	}
+
+	// Check semantic distance between new content and buffer summary
+	bufferSummary := strings.Join(d.buffer, " ")
+	dist := semanticDistance(bufferSummary, content)
+
+	d.buffer = append(d.buffer, content)
+
+	// Boundary detected if:
+	// 1. Semantic distance exceeds threshold (topic shift), OR
+	// 2. Buffer is full (forced consolidation)
+	if dist > d.threshold || len(d.buffer) >= d.maxBuffer {
+		return true
+	}
+	return false
+}
+
+// Flush returns and clears the current buffer.
+func (d *Detector) Flush() []string {
+	buf := d.buffer
+	d.buffer = nil
+	return buf
+}
+
+// Size returns current buffer size.
+func (d *Detector) Size() int { return len(d.buffer) }
+
+// semanticDistance computes a lightweight semantic distance between two texts.
+// Uses TF-IDF-inspired bag-of-words cosine distance — no embeddings needed.
+// Returns 0.0 (identical) to 1.0 (completely different).
+func semanticDistance(a, b string) float64 {
+	vecA := termFreq(a)
+	vecB := termFreq(b)
+	return 1.0 - cosineSim(vecA, vecB)
+}
+
+func termFreq(text string) map[string]float64 {
+	words := tokenize(text)
+	freq := map[string]float64{}
+	for _, w := range words {
+		if len(w) > 2 && !isStopWord(w) {
+			freq[w]++
+		}
+	}
+	// Normalize
+	total := 0.0
+	for _, v := range freq {
+		total += v * v
+	}
+	if total > 0 {
+		norm := math.Sqrt(total)
+		for k := range freq {
+			freq[k] /= norm
+		}
+	}
+	return freq
+}
+
+func cosineSim(a, b map[string]float64) float64 {
+	dot := 0.0
+	for k, va := range a {
+		if vb, ok := b[k]; ok {
+			dot += va * vb
+		}
+	}
+	return dot
+}
+
+func tokenize(text string) []string {
+	text = strings.ToLower(text)
+	var words []string
+	var word strings.Builder
+	for _, r := range text {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			word.WriteRune(r)
+		} else if word.Len() > 0 {
+			words = append(words, word.String())
+			word.Reset()
+		}
+	}
+	if word.Len() > 0 {
+		words = append(words, word.String())
+	}
+	return words
+}
+
+var stopWords = map[string]bool{
+	"the": true, "and": true, "for": true, "are": true, "but": true,
+	"not": true, "you": true, "all": true, "can": true, "had": true,
+	"her": true, "was": true, "one": true, "our": true, "out": true,
+	"use": true, "with": true, "this": true, "that": true, "from": true,
+	"they": true, "will": true, "have": true, "been": true, "when": true,
+}
+
+func isStopWord(w string) bool { return stopWords[w] }