Add observability for memory + agent-loop lifecycle signals (#18)

Author: jkyberneees

AGENTS.md

@@ -43,11 +43,11 @@ cmd/odek/
   *_test.go                   200+ unit + E2E tests covering all tools
 internal/
   llm/                        OpenAI-compatible HTTP client with reasoning_content support
-  loop/                       ReAct engine: observe → think → parallel-act → repeat
+  loop/                       ReAct engine: observe → think → parallel-act → repeat. signal.go — SignalEvent observability (context_trimmed, tool_recovery).
   tool/                       Thread-safe tool registry, clarify.go, send_message.go
   danger/                     Command/URL classification + bypass-resistant tokenizer (substitution, $IFS, wrappers, basenames). TTYApprover with friction mode.
   auth/                       Interactive approval system
-  memory/                     MemoryManager (facts, buffer, episodes, merge, scan). EpisodeProvenance — tainted episodes never auto-replayed.
+  memory/                     MemoryManager (facts, buffer, episodes, merge, scan). EpisodeProvenance — tainted episodes never auto-replayed. notifier.go — MemoryEvent lifecycle observability (fact/episode events fan out to terminal/WebUI/Telegram/programmatic handler).
   session/                    Session store (CRUD, trim, cleanup, compact JSON). AuditStore + divergence heuristic.
   skills/                     Skill system (types, loader, triggers, self-improve, curator, import, cache). SkillProvenance gate — NeedsReview skills pinned to Lazy.
   config/                     Config file loading, env vars, secrets.env, priority merge

README.md

@@ -45,7 +45,7 @@ Parallel OS-process sub-agents via `delegate_tasks`. True isolation — each sub
 Skill-matched `SKILL.md` files load on-demand. Auto-learns from patterns every session — detects multi-step procedures, error recoveries, repeated actions, and user corrections. **LLM-enhanced**: each detected pattern is enriched with an LLM-generated name, description, trigger keywords, and structured body with overview, steps, pitfalls, and verification sections. Use `--no-learn` to disable. Import skills from any URI with automatic LLM risk assessment. [docs/CLI.md#skills](docs/CLI.md#skills)
 
 ### 💾 Persistent Memory
-Three tiers: **facts** (agent-managed durable entries), **session buffer** (auto-appended turn summaries), **episodes** (LLM-extracted knowledge from past sessions). Merge-on-write via go-vector RandomProjections — cosine >0.7 auto-merges, <0.3 auto-adds. Saves ~80% LLM calls. [docs/MEMORY.md](docs/MEMORY.md)
+Three tiers: **facts** (agent-managed durable entries), **session buffer** (auto-appended turn summaries), **episodes** (LLM-extracted knowledge from past sessions). Merge-on-write via go-vector RandomProjections — cosine >0.7 auto-merges, <0.3 auto-adds. Saves ~80% LLM calls. Every lifecycle moment (fact add/merge/consolidate, episode store/dedup/evict/promote) emits an observable event surfaced in the terminal (verbose), Web UI, Telegram, or a programmatic `MemoryEventHandler`. [docs/MEMORY.md](docs/MEMORY.md)
 
 ### 🔧 Multi-Turn Sessions
 Save, resume, list, trim, and clean up conversations. Sessions persist as JSON in `~/.odek/sessions/`. Continue any session with `odek continue`. [docs/SESSIONS.md](docs/SESSIONS.md)

cmd/odek/main.go

@@ -827,6 +827,11 @@ func run(args []string) error {
 		rend.WithSkillVerbose(resolved.Skills.Verbose)
 	}
 
+	// Surface memory lifecycle + agent-signal notifications in verbose mode so
+	// fact/episode activity and silent recoveries (context trim, tool recovery)
+	// are observable without flooding the default terminal output.
+	rend.WithMemoryVerbose(resolved.InteractionMode == "verbose")
+
 	// Resolve skills config pointer (only when learn mode is enabled)
 	var skillsCfg *skills.SkillsConfig
 	if resolved.Skills.Learn {

cmd/odek/serve.go

@@ -384,6 +384,27 @@ func newServeAgent(resolved config.ResolvedConfig, system string, sendFn func(v
 				"heuristic":  event.Heuristic,
 			})
 		},
+		MemoryEventHandler: func(event memory.MemoryEvent) {
+			sendFn(map[string]any{
+				"type":       "memory_event",
+				"event":      event.Type,
+				"target":     event.Target,
+				"session_id": event.SessionID,
+				"content":    event.Content,
+				"count":      event.Count,
+				"new_count":  event.NewCount,
+				"untrusted":  event.Untrusted,
+			})
+		},
+		AgentSignalHandler: func(event loop.SignalEvent) {
+			sendFn(map[string]any{
+				"type":   "agent_signal",
+				"event":  event.Type,
+				"detail": event.Detail,
+				"tool":   event.Tool,
+				"count":  event.Count,
+			})
+		},
 		// Stream thinking/reasoning content to the WebUI.
 		// Only fire for pre-tool iterations (reasoning before tool calls);
 		// post-tool callbacks have no new reasoning to display.

cmd/odek/telegram.go

@@ -21,6 +21,7 @@ import (
 	"github.com/BackendStack21/odek/internal/config"
 	"github.com/BackendStack21/odek/internal/llm"
 	"github.com/BackendStack21/odek/internal/loop"
+	"github.com/BackendStack21/odek/internal/memory"
 	"github.com/BackendStack21/odek/internal/render"
 	"github.com/BackendStack21/odek/internal/schedule"
 	"github.com/BackendStack21/odek/internal/session"
@@ -1526,6 +1527,51 @@ func handleChatMessage(
 					&telegram.SendOpts{ReplyMarkup: replyMarkup, ParseMode: "Markdown", ReplyToMessageID: messageID})
 			}
 		},
+		MemoryEventHandler: func(event memory.MemoryEvent) {
+			// Only surface memory activity in verbose mode, and only the
+			// operationally meaningful events (skip high-frequency/internal ones
+			// like episode_stored and episode_deduped to avoid chat noise).
+			if skillsCfg == nil || !skillsCfg.Verbose {
+				return
+			}
+			var msg string
+			switch event.Type {
+			case "fact_added":
+				msg = "🧠 Memory fact added (" + event.Target + ")"
+			case "fact_merged":
+				msg = "🧠 Memory fact merged (" + event.Target + ")"
+			case "fact_replaced":
+				msg = "🧠 Memory fact updated (" + event.Target + ")"
+			case "fact_removed":
+				msg = "🧠 Memory fact removed (" + event.Target + ")"
+			case "fact_consolidated":
+				msg = fmt.Sprintf("🧠 Memory consolidated (%s: %d → %d)", event.Target, event.Count, event.NewCount)
+			case "episode_evicted":
+				msg = fmt.Sprintf("💾 %d episode(s) evicted", event.Count)
+			case "episode_pending_review":
+				msg = "🔒 Episode pending review (untrusted): " + event.SessionID
+			case "episode_promoted":
+				msg = "💾 Episode promoted: " + event.SessionID
+			default:
+				return
+			}
+			sendAsync(bot, chatID, msg, &telegram.SendOpts{ReplyToMessageID: messageID})
+		},
+		AgentSignalHandler: func(event loop.SignalEvent) {
+			if skillsCfg == nil || !skillsCfg.Verbose {
+				return
+			}
+			var msg string
+			switch event.Type {
+			case "context_trimmed":
+				msg = fmt.Sprintf("✂️ Context trimmed (%s): %d group(s) dropped", event.Detail, event.Count)
+			case "tool_recovery":
+				msg = "🔁 Tool recovery: " + event.Tool
+			default:
+				return
+			}
+			sendAsync(bot, chatID, msg, &telegram.SendOpts{ReplyToMessageID: messageID})
+		},
 		Approver:        approver,
 		DangerousConfig: &resolved.Dangerous,
 	}

cmd/odek/ui/app.js

@@ -316,6 +316,14 @@ function connect() {
       case 'skill_event':
         handleSkillEvent(event);
         break;
+
+      case 'memory_event':
+        handleMemoryEvent(event);
+        break;
+
+      case 'agent_signal':
+        handleAgentSignal(event);
+        break;
     }
   };
 }
@@ -1079,6 +1087,56 @@ function handleSkillEvent(event) {
   }
 }
 
+// ── Memory Events ──
+function handleMemoryEvent(event) {
+  switch (event.event) {
+    case 'fact_added':
+      showToast('🧠 Memory fact added (' + (event.target || '') + ')');
+      break;
+    case 'fact_merged':
+      showToast('🧠 Memory fact merged (' + (event.target || '') + ')');
+      break;
+    case 'fact_replaced':
+      showToast('🧠 Memory fact updated (' + (event.target || '') + ')');
+      break;
+    case 'fact_removed':
+      showToast('🧠 Memory fact removed (' + (event.target || '') + ')');
+      break;
+    case 'fact_consolidated':
+      showToast('🧠 Memory consolidated (' + (event.target || '') + ': ' +
+        (event.count || 0) + ' → ' + (event.new_count || 0) + ')');
+      break;
+    case 'episode_stored':
+      // Silent by default — fires after every qualifying session.
+      break;
+    case 'episode_promoted':
+      showToast('💾 ✓ Episode promoted: ' + (event.session_id || ''));
+      break;
+    case 'episode_evicted':
+      showToast('💾 ✗ ' + (event.count || 0) + ' episode(s) evicted');
+      break;
+    case 'episode_pending_review':
+      showToast('🔒 Episode pending review (untrusted): ' + (event.session_id || ''));
+      break;
+    case 'episode_deduped':
+      // Silent — internal dedup detail.
+      break;
+  }
+}
+
+// ── Agent Signals ──
+function handleAgentSignal(event) {
+  switch (event.event) {
+    case 'context_trimmed':
+      showToast('✂️ Context trimmed (' + (event.detail || '') + '): ' +
+        (event.count || 0) + ' group(s) dropped');
+      break;
+    case 'tool_recovery':
+      showToast('🔁 Tool recovery: ' + (event.tool || ''));
+      break;
+  }
+}
+
 // ── New Session ──
 // Saved on first load so we can restore the empty state after clearing.
 let savedEmptyStateNode = null;

docs/MEMORY.md

@@ -166,6 +166,39 @@ All memory content is scanned on write for:
 
 Rejected content returns an error to the agent.
 
+## Observability (lifecycle events)
+
+Every memory lifecycle moment emits a `memory.MemoryEvent` so operators can see
+activity that was previously silent. Events fan out (via `MultiMemoryNotifier`)
+to whichever surfaces are wired:
+
+- **Terminal** — shown in verbose interaction mode (`--interaction verbose`),
+  e.g. `🧠 memory[user] added: ...`, `🧠 consolidated memory[env] (5 → 2 entries)`.
+- **Web UI** — streamed over the WebSocket as `memory_event` and surfaced as toasts.
+- **Telegram** — posted in the chat when the bot runs verbose.
+- **Programmatic** — set `Config.MemoryEventHandler` to receive every event.
+
+| Event | Fired when | Key fields |
+|---|---|---|
+| `fact_added` | a new durable fact is appended (not on a silent dedup) | `Target`, `Content` |
+| `fact_merged` | merge-on-write folds a fact into a near-duplicate | `Target`, `Content`, `Similarity` |
+| `fact_replaced` | an existing fact is replaced | `Target`, `Content` |
+| `fact_removed` | a fact is removed | `Target`, `Content` |
+| `fact_consolidated` | LLM consolidation merges entries | `Target`, `Count`→`NewCount` |
+| `episode_stored` | a session episode is extracted + persisted | `SessionID`, `Count` (turns), `Untrusted` |
+| `episode_deduped` | a new episode replaces a near-duplicate | `SessionID`, `Similarity` |
+| `episode_evicted` | episodes pruned by TTL / count cap | `Sessions`, `Count` |
+| `episode_promoted` | a tainted episode is user-approved | `SessionID` |
+| `episode_pending_review` | an untrusted episode is stored but excluded from recall | `SessionID` |
+
+Notifiers must be non-blocking — fact writes fire mid-loop and episode events
+fire from the post-session background goroutines.
+
+The agent loop also emits `loop.SignalEvent`s for previously-silent self-healing
+(`context_trimmed` when message groups are dropped to fit the context window,
+`tool_recovery` when a repeatedly-failing tool triggers a corrective hint),
+surfaced the same way via `Config.AgentSignalHandler`.
+
 ## Architecture
 
 ### Episode Index Caching

internal/loop/loop.go

@@ -70,6 +70,7 @@ type Engine struct {
 	episodeCtx   EpisodeContextFunc // optional: per-turn episode search
 
 	toolEventHandler ToolEventHandler // optional: fires during tool execution
+	signalHandler    SignalHandler    // optional: fires on internal loop signals
 
 	// interactionMode controls how progress is surfaced to the user.
 	// "engaging" (default), "verbose", "enhance", or "off" (silent).
@@ -367,6 +368,12 @@ func (e *Engine) trimContext(messages []llm.Message, toolDefs []llm.ToolDef) []l
 		newMsgs = append(newMsgs, trimMsg)
 		newMsgs = append(newMsgs, messages[1:]...)
 		messages = newMsgs
+
+		e.emitSignal(SignalEvent{
+			Type:   "context_trimmed",
+			Detail: "proactive",
+			Count:  droppedGroups,
+		})
 	}
 
 	return messages
@@ -672,6 +679,11 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
 			if isContextLengthError(err) {
 				trimmed := trimToSurvival(messages)
 				if len(trimmed) < len(messages) {
+					e.emitSignal(SignalEvent{
+						Type:   "context_trimmed",
+						Detail: "survival",
+						Count:  len(messages) - len(trimmed),
+					})
 					messages = trimmed
 					// Reset memory index — trimToSurvival drops it.
 					e.memMsgIdx = -1
@@ -1029,6 +1041,11 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
 						toolName)
 				}
 				corrections = append(corrections, correction)
+				e.emitSignal(SignalEvent{
+					Type:   "tool_recovery",
+					Tool:   toolName,
+					Detail: correction,
+				})
 				// Reset counter after injecting suggestion
 				e.maxConsecutiveToolErrors[toolName] = 0
 			}

internal/loop/signal.go

@@ -0,0 +1,45 @@
+package loop
+
+import "time"
+
+// SignalEvent represents an internal agent-loop signal that was previously
+// invisible to the operator — moments where the engine silently intervened to
+// keep the session alive or productive. Surfacing these closes observability
+// gaps around context management and tool-failure recovery.
+//
+// Not every field is set for every Type; the zero value means "not applicable".
+type SignalEvent struct {
+	// Type is the signal kind. One of:
+	//   "context_trimmed"  — prior message groups were dropped to fit the token
+	//                        budget (Count = groups dropped, Detail = "proactive"
+	//                        for the pre-call budget trim or "survival" for the
+	//                        post-error nuclear trim)
+	//   "tool_recovery"    — a tool failed repeatedly and the engine injected a
+	//                        corrective hint so the model changes approach
+	//                        (Tool = failing tool, Detail = the correction)
+	Type      string
+	Detail    string    // human-readable detail (mode, correction text, etc.)
+	Tool      string    // tool name for tool_recovery
+	Count     int       // groups dropped (context_trimmed)
+	Timestamp time.Time // when the signal fired (UTC)
+}
+
+// SignalHandler receives agent-loop signal events. Implementations must be
+// non-blocking — signals fire inside the hot loop.
+type SignalHandler func(event SignalEvent)
+
+// emitSignal fires the engine's signal handler if one is configured, stamping
+// the timestamp when the caller left it zero. Safe to call unconditionally.
+func (e *Engine) emitSignal(ev SignalEvent) {
+	if e.signalHandler == nil {
+		return
+	}
+	if ev.Timestamp.IsZero() {
+		ev.Timestamp = time.Now().UTC()
+	}
+	e.signalHandler(ev)
+}
+
+// SetSignalHandler sets the optional agent-loop signal callback. Passing nil
+// disables signal emission.
+func (e *Engine) SetSignalHandler(cb SignalHandler) { e.signalHandler = cb }

internal/loop/signal_test.go

@@ -0,0 +1,42 @@
+package loop
+
+import "testing"
+
+func TestEmitSignal_NilHandlerIsSafe(t *testing.T) {
+	e := &Engine{}
+	// No handler set — must be a no-op, not a panic.
+	e.emitSignal(SignalEvent{Type: "context_trimmed"})
+}
+
+func TestSetSignalHandler_ReceivesEventsAndStampsTime(t *testing.T) {
+	e := &Engine{}
+	var got []SignalEvent
+	e.SetSignalHandler(func(ev SignalEvent) { got = append(got, ev) })
+
+	e.emitSignal(SignalEvent{Type: "context_trimmed", Detail: "survival", Count: 3})
+	e.emitSignal(SignalEvent{Type: "tool_recovery", Tool: "shell", Detail: "try a different approach"})
+
+	if len(got) != 2 {
+		t.Fatalf("expected 2 signals, got %d", len(got))
+	}
+	if got[0].Type != "context_trimmed" || got[0].Count != 3 || got[0].Detail != "survival" {
+		t.Errorf("unexpected first signal: %+v", got[0])
+	}
+	if got[0].Timestamp.IsZero() {
+		t.Error("expected timestamp to be stamped on emit")
+	}
+	if got[1].Tool != "shell" {
+		t.Errorf("expected tool=shell, got %q", got[1].Tool)
+	}
+}
+
+func TestSetSignalHandler_NilDisables(t *testing.T) {
+	e := &Engine{}
+	called := false
+	e.SetSignalHandler(func(SignalEvent) { called = true })
+	e.SetSignalHandler(nil)
+	e.emitSignal(SignalEvent{Type: "tool_recovery"})
+	if called {
+		t.Error("handler should not fire after being set to nil")
+	}
+}