feat(loop): prompt tiering — separate memory message for cache efficiency

Splits memory injection into a separate system message rather than
concatenating into messages[0] (baseSystem). This keeps the stable
base prompt identical across turns, enabling DeepSeek/Anthropic prompt
caching to reuse cached tokens.

Changes:
- Engine.memMsgIdx tracks volatile memory message position
- memoryPromptFunc output goes to messages[1] per-iteration
- messages[0] (baseSystem) never modified after init
- memMsgIdx resets between Run/RunWithMessages calls
- odek.go: removed init-time memory append (per-turn callback handles it)

Estimated token savings: ~60% of system prompt tokens cached
(baseSystem ~5K tokens cached, only memory block ~1K recalculated)

Author: molty3000

internal/loop/loop.go

@@ -72,6 +72,13 @@ type Engine struct {
 	// are visible to the agent on the next turn.
 	memoryPromptFunc func() string
 
+	// memMsgIdx tracks the position of the volatile memory system message
+	// in the messages array. -1 means not yet inserted. Using a separate
+	// message for memory (rather than concatenating into messages[0]) lets
+	// DeepSeek/Anthropic prompt caching keep the stable baseSystem cached
+	// across turns — only the memory message changes each iteration.
+	memMsgIdx int
+
 	// PromptCaching enables Anthropic/OpenAI/DeepSeek prompt caching markers.
 	// When enabled, the system prompt and first user message are annotated
 	// with cache_control markers, and the system prompt is moved to the
@@ -288,6 +295,7 @@ func (e *Engine) trimContext(messages []llm.Message, toolDefs []llm.ToolDef) []l
 
 // Run executes the loop for a given task and returns the final response.
 func (e *Engine) Run(ctx context.Context, task string) (string, error) {
+	e.memMsgIdx = -1
 	messages := []llm.Message{
 		{Role: "user", Content: task},
 	}
@@ -308,6 +316,7 @@ func (e *Engine) Run(ctx context.Context, task string) (string, error) {
 // new user message, call RunWithMessages, then save the returned messages.
 func (e *Engine) RunWithMessages(ctx context.Context, messages []llm.Message) (string, []llm.Message, error) {
 	// Reset token accounting for this run
+	e.memMsgIdx = -1
 	e.TotalInputTokens = 0
 	e.TotalOutputTokens = 0
 	e.TotalCacheCreationTokens = 0
@@ -401,13 +410,30 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
 
 		// Refresh memory content before each LLM call so the agent sees
 		// the latest facts even if it mutated memory during this session.
+		// Memory is injected as a separate system message (messages[1] or
+		// later) so that messages[0] (baseSystem) remains stable across
+		// turns — letting DeepSeek/Anthropic prompt caching keep it cached.
 		if e.memoryPromptFunc != nil {
 			if memBlock := e.memoryPromptFunc(); memBlock != "" {
-				e.system = e.baseSystem + memBlock
-				// Update messages[0] if it's the system message
+				// Keep messages[0] as the stable baseSystem (never modified).
 				if len(messages) > 0 && messages[0].Role == "system" {
-					messages[0].Content = e.system
+					messages[0].Content = e.baseSystem
+				}
+				memMsg := llm.Message{Role: "system", Content: memBlock}
+				if e.memMsgIdx >= 0 && e.memMsgIdx < len(messages) {
+					// Update existing memory slot — keeps position stable.
+					messages[e.memMsgIdx].Content = memBlock
+				} else {
+					// First time: insert memory message after base system.
+					insertAt := 1
+					messages = append(messages[:insertAt],
+						append([]llm.Message{memMsg}, messages[insertAt:]...)...)
+					e.memMsgIdx = insertAt
 				}
+			} else if e.memMsgIdx >= 0 && e.memMsgIdx < len(messages) {
+				// No memory block — remove the memory message if present.
+				messages = append(messages[:e.memMsgIdx], messages[e.memMsgIdx+1:]...)
+				e.memMsgIdx = -1
 			}
 		}
 

internal/loop/loop_test.go

@@ -941,3 +941,168 @@ func TestEngine_Run_CacheAccumulation_NoCache(t *testing.T) {
 		t.Errorf("TotalCachedTokens = %d, want 0", engine.TotalCachedTokens)
 	}
 }
+
+// ── Prompt Tiering Tests ───────────────────────────────────────────
+
+// TestPromptTiering_SeparateMemoryMessage verifies that memory is injected
+// as a separate system message rather than concatenated into messages[0].
+// This ensures messages[0] (baseSystem) remains stable across turns for
+// DeepSeek/Anthropic prompt caching.
+func TestPromptTiering_SeparateMemoryMessage(t *testing.T) {
+	callCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		callCount++
+		var body struct {
+			Messages []struct {
+				Role    string `json:"role"`
+				Content string `json:"content"`
+			} `json:"messages"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			return
+		}
+
+		if callCount == 1 {
+			// Verify: messages[0] = baseSystem (stable), messages[1] = memory (volatile)
+			if len(body.Messages) < 2 {
+				t.Errorf("expected at least 2 messages (system + memory + user), got %d", len(body.Messages))
+			} else {
+				if body.Messages[0].Role != "system" {
+					t.Errorf("messages[0].Role = %q, want system", body.Messages[0].Role)
+				}
+				if body.Messages[0].Content != "You are a stable base." {
+					t.Errorf("messages[0].Content = %q, want %q", body.Messages[0].Content, "You are a stable base.")
+				}
+				if body.Messages[1].Role != "system" {
+					t.Errorf("messages[1].Role = %q, want system (memory)", body.Messages[1].Role)
+				}
+				if body.Messages[1].Content != "memory-block-v1" {
+					t.Errorf("messages[1].Content = %q, want memory-block-v1", body.Messages[1].Content)
+				}
+			}
+			// Return a tool call to force another iteration.
+			fmt.Fprint(w, `{"choices":[{"message":{"content":"","tool_calls":[{"id":"call_1","function":{"name":"echo","arguments":"{}"}}]}}]}`)
+		} else {
+			// Second call: memory should be updated.
+			if len(body.Messages) >= 2 && body.Messages[1].Role == "system" {
+				if body.Messages[1].Content != "memory-block-v2" {
+					t.Errorf("messages[1].Content = %q, want memory-block-v2", body.Messages[1].Content)
+				}
+				// messages[0] must still be the stable base.
+				if body.Messages[0].Content != "You are a stable base." {
+					t.Errorf("messages[0].Content changed: %q, want %q", body.Messages[0].Content, "You are a stable base.")
+				}
+			}
+			fmt.Fprint(w, `{"choices":[{"message":{"content":"done"}}]}`)
+		}
+	}))
+	defer server.Close()
+
+	client := llm.New(server.URL, "sk-test", "test-model", "", 0)
+	echoTool := &fakeTool{name: "echo", description: "echo", output: "ok"}
+	registry := tool.NewRegistry([]tool.Tool{echoTool})
+	engine := New(client, registry, 10, "You are a stable base.", nil, 0)
+
+	// Set up memory callback that returns different values per call.
+	memVersion := 0
+	engine.SetMemoryPromptFunc(func() string {
+		memVersion++
+		return fmt.Sprintf("memory-block-v%d", memVersion)
+	})
+
+	_, err := engine.Run(context.Background(), "test")
+	if err != nil {
+		t.Fatalf("Run() error: %v", err)
+	}
+}
+
+// TestPromptTiering_NoMemoryDropsMessage verifies that when the memory
+// callback returns empty, the memory system message is removed.
+func TestPromptTiering_NoMemoryDropsMessage(t *testing.T) {
+	callCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		callCount++
+		var body struct {
+			Messages []struct {
+				Role    string `json:"role"`
+				Content string `json:"content"`
+			} `json:"messages"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			return
+		}
+
+		if callCount == 1 {
+			// First call: memory is non-empty, should be at index 1.
+			if len(body.Messages) >= 2 && body.Messages[1].Role == "system" {
+				if body.Messages[1].Content != "initial-memory" {
+					t.Errorf("unexpected memory: %q", body.Messages[1].Content)
+				}
+			}
+			fmt.Fprint(w, `{"choices":[{"message":{"content":"","tool_calls":[{"id":"call_1","function":{"name":"echo","arguments":"{}"}}]}}]}`)
+		} else {
+			// Second call: memory is empty, should NOT have a second system message.
+			systemCount := 0
+			for _, m := range body.Messages {
+				if m.Role == "system" {
+					systemCount++
+				}
+			}
+			if systemCount != 1 {
+				t.Errorf("expected 1 system message (base only), got %d", systemCount)
+			}
+			fmt.Fprint(w, `{"choices":[{"message":{"content":"done"}}]}`)
+		}
+	}))
+	defer server.Close()
+
+	client := llm.New(server.URL, "sk-test", "test-model", "", 0)
+	echoTool := &fakeTool{name: "echo", description: "echo", output: "ok"}
+	registry := tool.NewRegistry([]tool.Tool{echoTool})
+	engine := New(client, registry, 10, "You are a stable base.", nil, 0)
+
+	memVersion := 0
+	engine.SetMemoryPromptFunc(func() string {
+		memVersion++
+		if memVersion == 1 {
+			return "initial-memory"
+		}
+		return "" // Empty after first call
+	})
+
+	_, err := engine.Run(context.Background(), "test")
+	if err != nil {
+		t.Fatalf("Run() error: %v", err)
+	}
+}
+
+// TestPromptTiering_MemMsgIdxResets verifies that memMsgIdx resets
+// between Run calls, preventing stale index carry-over.
+func TestPromptTiering_MemMsgIdxResets(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `{"choices":[{"message":{"content":"ok"}}]}`)
+	}))
+	defer server.Close()
+
+	client := llm.New(server.URL, "sk-test", "test-model", "", 0)
+	engine := New(client, registryOrNil(), 10, "base system", nil, 0)
+
+	// Run 1 with memory
+	engine.SetMemoryPromptFunc(func() string { return "mem-run1" })
+	engine.Run(context.Background(), "run1")
+
+	// memMsgIdx should be set
+	if engine.memMsgIdx != 1 {
+		t.Errorf("after run1: memMsgIdx = %d, want 1", engine.memMsgIdx)
+	}
+
+	// Run 2 without memory callback — should reset
+	engine.memoryPromptFunc = nil
+	engine.Run(context.Background(), "run2")
+
+	if engine.memMsgIdx != -1 {
+		t.Errorf("after run2 (no callback): memMsgIdx = %d, want -1", engine.memMsgIdx)
+	}
+}
+
+func registryOrNil() *tool.Registry { return tool.NewRegistry(nil) }

odek.go

@@ -435,12 +435,8 @@ func New(cfg Config) (*Agent, error) {
 	}
 
 	// Wire per-turn memory injection so the agent sees the latest facts
-	// even after mutating memory during a session.
-	// The initial system message gets the current memory snapshot,
 	// and the loop engine refreshes it before each LLM call.
-	if memoryBlock := memoryManager.BuildSystemPrompt(); memoryBlock != "" {
-		cfg.SystemMessage += memoryBlock
-	}
+	// (Memory is injected per-turn via SetMemoryPromptFunc below.)
 
 	// Auto-search relevant past episodes and inject them into the system
 	// prompt so the agent has context from previous sessions without