feat: 8 improvements from top 20 OSS comparison

Based on deep analysis of Mem0, Letta, Cognee, memU, MemOS, Hindsight,
Engram, agentmemory, Zep, and memvid:

1. Memory conflict resolution (Mem0-inspired)
   - Detect contradictions: same topic, different facts
   - Auto-supersede old with new, lower old confidence
   - Audit trail via node_versions

2. Memory compaction (Engram/Letta-inspired)
   - Auto-summarize when graph exceeds token budget
   - Merge low-confidence, rarely-accessed nodes by type
   - Keep anchors (file/entity/session) intact

3. Temporal backbone (MAGMA/Zep Graphiti-inspired)
   - Immutable timeline chain per project
   - Every node auto-linked to previous via learned_in edge
   - Timeline traversal: walk forward/backward

4. Dedup rolling window (Engram/agentmemory-inspired)
   - Skip near-duplicates within configurable time window (default 5min)
   - SHA-256 hash with whitespace normalization

5. Topic dedup (Engram-inspired)
   - UpsertByTopic: same project+scope+topic_key updates existing node
   - Version history preserved on update

6. SQLite encryption at rest (Engram E2E-inspired)
   - AES-256-GCM file-level encryption
   - GenerateKey, EncryptFile, DecryptFile, IsEncrypted

7. Mental models (Hindsight-inspired)
   - Auto-generated project summary from high-confidence nodes
   - Stack detection, conventions, active tasks, key decisions
   - Format() returns markdown for agent injection

8. TypeScript SDK (@graycode/yaad)
   - Full REST API wrapper: remember, recall, context, link, impact
   - Session management, feedback, SSE events
   - Ready for npm publish

Author: Patel230

internal/compact/compact.go

@@ -0,0 +1,120 @@
+// Package compact implements memory compaction — auto-summarize when
+// the graph exceeds a token budget. Based on Engram and Letta approaches.
+package compact
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/GrayCodeAI/yaad/internal/storage"
+)
+
+// Compactor summarizes old, low-confidence memories to keep the graph lean.
+type Compactor struct {
+	store     *storage.Store
+	maxTokens int // max total tokens before compaction triggers
+}
+
+func New(store *storage.Store, maxTokens int) *Compactor {
+	if maxTokens <= 0 {
+		maxTokens = 50000 // ~200KB of content
+	}
+	return &Compactor{store: store, maxTokens: maxTokens}
+}
+
+// NeedsCompaction returns true if total content exceeds the token budget.
+func (c *Compactor) NeedsCompaction(project string) (bool, int) {
+	nodes, _ := c.store.ListNodes(storage.NodeFilter{Project: project})
+	totalTokens := 0
+	for _, n := range nodes {
+		totalTokens += len(n.Content) / 4 // ~4 chars per token
+	}
+	return totalTokens > c.maxTokens, totalTokens
+}
+
+// Compact merges low-confidence, old memories of the same type into summary nodes.
+// Returns the number of nodes compacted.
+func (c *Compactor) Compact(project string) (int, error) {
+	nodes, err := c.store.ListNodes(storage.NodeFilter{Project: project})
+	if err != nil {
+		return 0, err
+	}
+
+	// Group by type
+	byType := map[string][]*storage.Node{}
+	for _, n := range nodes {
+		if n.Type == "file" || n.Type == "entity" || n.Type == "session" {
+			continue // don't compact anchors or sessions
+		}
+		if n.Confidence < 0.5 && n.AccessCount < 3 {
+			byType[n.Type] = append(byType[n.Type], n)
+		}
+	}
+
+	compacted := 0
+	for typ, group := range byType {
+		if len(group) < 3 {
+			continue // not enough to compact
+		}
+
+		// Build summary from group
+		var contents []string
+		var ids []string
+		for _, n := range group {
+			contents = append(contents, n.Content)
+			ids = append(ids, n.ID)
+		}
+
+		summary := buildCompactSummary(typ, contents)
+
+		// Create summary node
+		summaryNode := &storage.Node{
+			ID:          uuid.New().String(),
+			Type:        typ,
+			Content:     summary,
+			ContentHash: fmt.Sprintf("compact:%s:%d", typ, len(ids)),
+			Summary:     fmt.Sprintf("Compacted %d %s memories", len(ids), typ),
+			Scope:       group[0].Scope,
+			Project:     project,
+			Tier:        3, // cold
+			Confidence:  0.6,
+			Version:     1,
+		}
+		if err := c.store.CreateNode(summaryNode); err != nil {
+			continue
+		}
+
+		// Archive compacted nodes
+		for _, id := range ids {
+			old, _ := c.store.GetNode(id)
+			if old != nil {
+				c.store.SaveVersion(old.ID, old.Content, "compactor", "compacted into "+summaryNode.ID[:8])
+				old.Confidence = 0
+				c.store.UpdateNode(old)
+				compacted++
+			}
+		}
+	}
+	return compacted, nil
+}
+
+func buildCompactSummary(typ string, contents []string) string {
+	// Take first 5 items as representative
+	limit := 5
+	if len(contents) < limit {
+		limit = len(contents)
+	}
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("Summary of %d %s memories:\n", len(contents), typ))
+	for i, c := range contents[:limit] {
+		if len(c) > 100 {
+			c = c[:100] + "..."
+		}
+		sb.WriteString(fmt.Sprintf("%d. %s\n", i+1, c))
+	}
+	if len(contents) > limit {
+		sb.WriteString(fmt.Sprintf("... and %d more\n", len(contents)-limit))
+	}
+	return sb.String()
+}

internal/conflict/resolver.go

@@ -0,0 +1,129 @@
+// Package conflict detects and resolves contradictory memories.
+// When a new memory contradicts an existing one (same entity/topic, different fact),
+// the old memory is auto-superseded. Based on Mem0's conflict resolution approach.
+package conflict
+
+import (
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/GrayCodeAI/yaad/internal/storage"
+)
+
+// Resolver detects and resolves memory conflicts.
+type Resolver struct {
+	store *storage.Store
+}
+
+func New(store *storage.Store) *Resolver {
+	return &Resolver{store: store}
+}
+
+// CheckAndResolve checks if a new node contradicts existing nodes.
+// If a contradiction is found, creates a supersedes edge and lowers the old node's confidence.
+// Returns the list of superseded node IDs.
+func (r *Resolver) CheckAndResolve(newNode *storage.Node) ([]string, error) {
+	// Find existing nodes of the same type in the same project
+	existing, err := r.store.ListNodes(storage.NodeFilter{
+		Type:    newNode.Type,
+		Project: newNode.Project,
+		Scope:   newNode.Scope,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var superseded []string
+	for _, old := range existing {
+		if old.ID == newNode.ID || old.Confidence <= 0 {
+			continue
+		}
+		if isContradiction(newNode, old) {
+			// Create supersedes edge
+			r.store.CreateEdge(&storage.Edge{
+				ID:      uuid.New().String(),
+				FromID:  newNode.ID,
+				ToID:    old.ID,
+				Type:    "supersedes",
+				Acyclic: true,
+				Weight:  1.0,
+			})
+			// Lower old node confidence
+			old.Confidence *= 0.3
+			r.store.UpdateNode(old)
+			// Save version for audit trail
+			r.store.SaveVersion(old.ID, old.Content, "conflict-resolver",
+				"superseded by "+newNode.ID[:8])
+			superseded = append(superseded, old.ID)
+		}
+	}
+	return superseded, nil
+}
+
+// isContradiction detects if two nodes about the same topic say different things.
+func isContradiction(newNode, oldNode *storage.Node) bool {
+	// Extract key entities from both
+	newEntities := extractKeyTerms(newNode.Content)
+	oldEntities := extractKeyTerms(oldNode.Content)
+
+	// Must share at least 2 key terms (same topic)
+	shared := 0
+	for term := range newEntities {
+		if oldEntities[term] {
+			shared++
+		}
+	}
+	if shared < 2 {
+		return false // different topics, not a contradiction
+	}
+
+	// Check for negation patterns
+	newLower := strings.ToLower(newNode.Content)
+
+	// "Use X" vs "Don't use X" or "Use Y instead of X"
+	if strings.Contains(newLower, "instead of") || strings.Contains(newLower, "not ") ||
+		strings.Contains(newLower, "replaced") || strings.Contains(newLower, "switched") ||
+		strings.Contains(newLower, "migrated") || strings.Contains(newLower, "changed") {
+		return true
+	}
+
+	// "Chose X" vs "Chose Y" for same topic (shared entities but different choice)
+	if newNode.Type == "decision" && oldNode.Type == "decision" && shared >= 2 {
+		// Different content but same topic = likely updated decision
+		if newNode.Content != oldNode.Content {
+			return true
+		}
+	}
+
+	// Same convention type, same entities, different content = updated convention
+	if newNode.Type == "convention" && oldNode.Type == "convention" && shared >= 2 {
+		if newNode.Content != oldNode.Content {
+			return true
+		}
+	}
+
+	return false
+}
+
+func extractKeyTerms(content string) map[string]bool {
+	terms := map[string]bool{}
+	words := strings.Fields(strings.ToLower(content))
+	for _, w := range words {
+		w = strings.Trim(w, ".,;:!?\"'()[]{}") 
+		if len(w) > 3 && !isStopWord(w) {
+			terms[w] = true
+		}
+	}
+	return terms
+}
+
+var stopWords = map[string]bool{
+	"the": true, "and": true, "for": true, "are": true, "but": true,
+	"not": true, "you": true, "all": true, "can": true, "had": true,
+	"was": true, "one": true, "our": true, "use": true, "with": true,
+	"this": true, "that": true, "from": true, "they": true, "will": true,
+	"have": true, "been": true, "should": true, "would": true, "could": true,
+	"also": true, "than": true, "then": true, "into": true, "over": true,
+}
+
+func isStopWord(w string) bool { return stopWords[w] }

internal/dedup/window.go

@@ -0,0 +1,75 @@
+// Package dedup implements rolling-window deduplication.
+// Skips near-duplicate memories within a configurable time window.
+// Based on Engram's SHA-256 dedup with 15-minute rolling window
+// and agentmemory's 5-minute dedup window.
+package dedup
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Window tracks recent content hashes to skip near-duplicates.
+type Window struct {
+	duration time.Duration
+	seen     map[string]time.Time
+	mu       sync.Mutex
+}
+
+// New creates a dedup window. Default: 5 minutes.
+func New(duration time.Duration) *Window {
+	if duration <= 0 {
+		duration = 5 * time.Minute
+	}
+	return &Window{duration: duration, seen: map[string]time.Time{}}
+}
+
+// IsDuplicate returns true if content was seen within the rolling window.
+// If not a duplicate, records it for future checks.
+func (w *Window) IsDuplicate(content string) bool {
+	hash := contentHash(content)
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	// Clean expired entries
+	now := time.Now()
+	for k, t := range w.seen {
+		if now.Sub(t) > w.duration {
+			delete(w.seen, k)
+		}
+	}
+
+	// Check if seen recently
+	if _, exists := w.seen[hash]; exists {
+		return true
+	}
+
+	w.seen[hash] = now
+	return false
+}
+
+// NormalizedHash returns a hash that ignores whitespace differences.
+func contentHash(content string) string {
+	h := sha256.Sum256([]byte(normalizeWhitespace(content)))
+	return fmt.Sprintf("%x", h[:16])
+}
+
+func normalizeWhitespace(s string) string {
+	var result []byte
+	space := false
+	for _, b := range []byte(s) {
+		if b == ' ' || b == '\t' || b == '\n' || b == '\r' {
+			if !space {
+				result = append(result, ' ')
+				space = true
+			}
+		} else {
+			result = append(result, b)
+			space = false
+		}
+	}
+	return string(result)
+}