feat: tool compression, episode semantic search, conversation skill extraction (v0.27.0)

A — Auto-skill generation from conversations:
- ExtractSkillsFromConversation: new LLM-based heuristic that analyzes the
  full conversation history (user intent, agent reasoning, tool calls,
  final outcome) to discover reusable skills. Unlike pattern-based
  heuristics (multi-step, error-recovery), this catches architectural
  decisions, debugging strategies, and workflow patterns.
- Called from learnAndSuggest alongside existing heuristics when LLM
  learning is enabled.

B — Episode semantic search:
- NewRPRanker: uses go-vector RandomProjections for cosine similarity
  ranking of episodes. Replaces recency-based default ranking with
  actual semantic matching — 'bug we fixed last week' finds the right
  episode. No LLM cost, <1ms per search.
- Now the default ranker when LLM search is not explicitly enabled.

C — Tool output compression:
- Large tool outputs (>4KB) are automatically truncated to head+tail
  with an omission notice. Keeps 3KB head + 1KB tail, preserving the
  most important information while saving context window for reasoning.

Author: molty3000

cmd/odek/main.go

@@ -1312,6 +1312,22 @@ func learnAndSuggest(messages []llm.Message, sm *skills.SkillManager, llmClient
 	userMessages := extractUserMessages(messages)
 	suggestions := skills.RunAllHeuristics(skillMsgs, userMessages)
 
+	// Conversation-level skill extraction — uses full context, not just tool patterns.
+	// Catches architectural decisions, debugging strategies, and workflow patterns
+	// that the pattern-based heuristics miss.
+	if llmLearn && llmClient != nil {
+		if convSkill := skills.ExtractSkillsFromConversation(llmClient, skillMsgs, userMessages); convSkill != nil {
+			// Build a command log from tool calls for context
+			calls := skills.ExtractToolCalls(skillMsgs)
+			cmds := make([]string, 0, len(calls))
+			for _, c := range calls {
+				cmds = append(cmds, c.Input)
+			}
+			convSkill.CommandLog = cmds
+			suggestions = append(suggestions, *convSkill)
+		}
+	}
+
 	// Apply LLM enhancement to each suggestion
 	for i := range suggestions {
 		if llmLearn && llmClient != nil {

internal/loop/loop.go

@@ -523,7 +523,20 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
 				e.toolEventHandler("tool_result", tc.Function.Name, output)
 			}
 
-			// Wrap tool output in unbreakable delimiters so the model
+			// Compress large tool outputs to save context window.
+		// Keep the first and last portions — head usually contains
+		// the most important info, tail may have final results.
+		const maxOutput = 4096
+		if len(output) > maxOutput {
+			head := maxOutput * 3 / 4 // 3KB head
+			tail := maxOutput / 4     // 1KB tail
+			output = output[:head] +
+				fmt.Sprintf("\n\n... [%d bytes omitted — output was %d bytes total] ...\n\n",
+					len(output)-head-tail, len(output)) +
+				output[len(output)-tail:]
+		}
+
+		// Wrap tool output in unbreakable delimiters so the model
 			// treats it as DATA, never as instructions. The header and
 			// footer both explicitly frame the content as untrusted data.
 			// Even if the output contains "ignore previous instructions",

internal/memory/episodes.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
+
+	"github.com/BackendStack21/go-vector/pkg/vector"
 )
 
 // maxEpisodeSummaryBytes caps how much summary text we store per episode.
@@ -248,5 +251,78 @@ func NewLLMRanker(llm LLMClient) RankStrategy {
 		return defaultRanker(query, episodes)
 	}
 	return ranked, nil
+	}
+}
+
+// NewRPRanker creates a RankStrategy that uses RandomProjections (go-vector)
+// for semantic similarity search over episodes. No LLM calls — pure vector
+// math. Falls back to recency ordering if RP fitting fails.
+//
+// The ranker fits the RP embedder on episode summaries on each call.
+// With the typical number of episodes (< 100, 120-char summaries each),
+// fitting takes < 1ms and is negligible compared to LLM latency.
+func NewRPRanker(dims int) RankStrategy {
+	if dims <= 0 {
+		dims = 64 // lower dims = faster, fine for short summaries
+	}
+	return func(query string, episodes []EpisodeMeta) ([]EpisodeMeta, error) {
+		if len(episodes) <= 1 {
+			out := make([]EpisodeMeta, len(episodes))
+			copy(out, episodes)
+			return out, nil
+		}
+
+		// Build corpus from episode summaries
+		corpus := make([]string, len(episodes))
+		for i, ep := range episodes {
+			corpus[i] = ep.Summary
+		}
+
+		// Fit RP and embed query + corpus
+		rp := vector.NewRandomProjections(dims)
+		rp.Fit(append(corpus, query))
+		queryVec, _ := rp.Embed(query)
+
+		// Score each episode by cosine similarity
+		type scored struct {
+			idx   int
+			score float32
+		}
+		scores := make([]scored, len(episodes))
+		for i, summary := range corpus {
+			vec, _ := rp.Embed(summary)
+			sim := cosineVector(queryVec, vec)
+			scores[i] = scored{idx: i, score: sim}
+		}
+
+		// Sort by score descending
+		sort.Slice(scores, func(i, j int) bool {
+			return scores[i].score > scores[j].score
+		})
+
+		out := make([]EpisodeMeta, len(episodes))
+		for i, s := range scores {
+			out[i] = episodes[s.idx]
+		}
+		return out, nil
+	}
 }
+
+// cosineVector computes cosine similarity between two go-vector Vectors.
+func cosineVector(a, b vector.Vector) float32 {
+	if len(a) != len(b) || len(a) == 0 {
+		return 0
+	}
+	var dot, normA, normB float64
+	for i := range a {
+		da := float64(a[i])
+		db := float64(b[i])
+		dot += da * db
+		normA += da * da
+		normB += db * db
+	}
+	if normA == 0 || normB == 0 {
+		return 0
+	}
+	return float32(dot / (math.Sqrt(normA) * math.Sqrt(normB)))
 }

internal/memory/memory.go

@@ -135,10 +135,13 @@ func NewMemoryManager(memoryDir string, llc LLMClient, cfg MemoryConfig) *Memory
 	episodesDir := memoryDir
 
 	factStore := NewFactStore(factsDir, cfg.FactsLimitUser, cfg.FactsLimitEnv)
-	// Use LLM-based episode ranker when an LLM client is available and enabled
+	// Use LLM-based episode ranker when an LLM client is available and enabled.
+	// Otherwise use RP (RandomProjections) semantic similarity — fast, no LLM cost.
 	var rankFn RankStrategy
 	if llc != nil && cfg.LLMSearch != nil && *cfg.LLMSearch {
 		rankFn = NewLLMRanker(llc)
+	} else {
+		rankFn = NewRPRanker(64)
 	}
 	episodeStore := NewEpisodeStore(episodesDir, rankFn)
 	mergeDetector := NewMergeDetectorWithThresholds(0, cfg.MergeThreshold, cfg.AddThreshold)

internal/skills/llm_enhance.go

@@ -163,3 +163,62 @@ func EnhanceCurationWithLLM(llm LLMClient, report *CurationReport) string {
 	}
 	return resp
 }
+
+// ExtractSkillsFromConversation takes the full conversation history (all messages)
+// and asks the LLM to identify whether a reusable skill was demonstrated.
+// Unlike GenerateSkillWithLLM (which only enhances pattern-detected tool call
+// sequences), this analyzes the complete interaction — user intent, agent
+// reasoning, tool calls, and final outcome — to discover deeper patterns.
+//
+// Returns nil if the LLM call fails or no skill is found.
+func ExtractSkillsFromConversation(llm LLMClient, messages []LlmMessage, userMessages []string) *SkillSuggestion {
+	if llm == nil || len(messages) < 3 {
+		return nil
+	}
+
+	var b strings.Builder
+	b.WriteString("Analyze this AI coding agent conversation for reusable skills:\n\n")
+
+	// Include up to 20 messages (user/assistant/tool) to give full context
+	maxMsgs := len(messages)
+	if maxMsgs > 20 {
+		maxMsgs = 20
+	}
+	start := len(messages) - maxMsgs
+	for i := start; i < len(messages); i++ {
+		m := messages[i]
+		content := m.Content
+		if len(content) > 300 {
+			content = content[:297] + "..."
+		}
+		label := m.Role
+		if m.Name != "" {
+			label = m.Role + ":" + m.Name
+		}
+		b.WriteString(fmt.Sprintf("[%s] %s\n", label, content))
+	}
+
+	b.WriteString("\nDid the agent demonstrate a reusable skill or procedure? ")
+	b.WriteString("If YES, generate a skill file. If NO, respond with just 'none'.\n\n")
+	b.WriteString("Output format:\n")
+	b.WriteString("NAME: <short kebab-case name>\n")
+	b.WriteString("DESCRIPTION: <one-line, max 100 chars>\n")
+	b.WriteString("TOPICS: <3-5 comma-separated topic keywords>\n")
+	b.WriteString("ACTIONS: <2-3 comma-separated action keywords>\n")
+	b.WriteString("BODY:\n")
+	b.WriteString("<markdown body with ## Overview, ## Step-by-Step, ## Common Pitfalls, ## Verification>\n")
+
+	resp, err := llm.SimpleCall(context.Background(),
+		"You are a skill curator. Given a conversation trace, identify if a reusable skill or procedure was demonstrated. If yes, generate a structured SKILL.md. If no clear reusable pattern exists, respond with just 'none'. Be conservative — only extract genuinely reusable skills.",
+		b.String(),
+	)
+	if err != nil || resp == "" || strings.TrimSpace(resp) == "none" {
+		return nil
+	}
+
+	s := parseLLMSuggestion(resp)
+	if s != nil {
+		s.Heuristic = "conversation-extracted"
+	}
+	return s
+}