feat(telemetry): JSONL search metrics in .ragcode/ (#6 from proposals)

- pkg/telemetry: AppendSearchMetric() — thread-safe JSONL append
  per tool invocation to {workspaceRoot}/.ragcode/search_metrics.jsonl
- pkg/telemetry: ReadAggregatedMetrics() — reads JSONL, returns
  cumulative stats (total searches, fallback vs vector, bytes saved, avg scores)
- smart_search: records metric async (goroutine) after each search
- 4 unit tests for write/read round-trip and edge cases

Author: razvan

internal/service/tools/smart_search.go

@@ -99,6 +99,7 @@ func (t *SmartSearchTool) Register(server *mcp.Server) {
 }
 
 func (t *SmartSearchTool) Execute(ctx context.Context, input SmartSearchInput) (string, error) {
+	t0 := time.Now()
 	query, limit, input, err := normalizeInput(input, t.searchLimit)
 	if err != nil {
 		return "", err
@@ -130,6 +131,9 @@ func (t *SmartSearchTool) Execute(ctx context.Context, input SmartSearchInput) (
 	response := t.buildResponseMeta(sr.meta, useCompact)
 	serializeResults(&response, merged, useCompact, isFallback, query, input.IncludeReasons)
 
+	// Record metric asynchronously to avoid blocking response
+	go recordSearchMetric(sr.meta, query, merged, isFallback, response.Context.Telemetry, t0)
+
 	return response.JSON()
 }
 

internal/service/tools/smart_search_pipeline.go

@@ -381,3 +381,33 @@ func noResultsResponse(query string, meta searchMetadata) (string, error) {
 	}
 	return response.JSON()
 }
+
+// recordSearchMetric maps pipeline data to a telemetry.SearchMetric and appends to JSONL.
+func recordSearchMetric(meta searchMetadata, query string, merged []mergedResult, isFallback bool, savings *telemetry.Savings, start time.Time) {
+	source := "vector"
+	if isFallback {
+		source = "fallback"
+	}
+
+	var topScore float32
+	if len(merged) > 0 {
+		topScore = merged[0].score
+	}
+
+	var bytesSaved, tokensSaved int64
+	if savings != nil {
+		bytesSaved = savings.BytesAvoided
+		tokensSaved = savings.TokensSaved
+	}
+
+	telemetry.AppendSearchMetric(meta.workspaceRoot, telemetry.SearchMetric{
+		Tool:        "rag_search",
+		Query:       query,
+		ResultCount: len(merged),
+		TopScore:    topScore,
+		Source:      source,
+		BytesSaved:  bytesSaved,
+		TokensSaved: tokensSaved,
+		ResponseMs:  time.Since(start).Milliseconds(),
+	})
+}

pkg/telemetry/metrics.go

@@ -0,0 +1,57 @@
+package telemetry
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// SearchMetric records a single tool invocation for cumulative analytics.
+type SearchMetric struct {
+	Timestamp    time.Time `json:"ts"`
+	Tool         string    `json:"tool"`                    // "rag_search", "rag_find_usages", etc.
+	Query        string    `json:"query,omitempty"`         // search query
+	ResultCount  int       `json:"result_count"`            // number of results returned
+	TopScore     float32   `json:"top_score,omitempty"`     // score of best result
+	Source       string    `json:"source,omitempty"`        // "vector", "fallback", "hybrid"
+	BytesSaved   int64     `json:"bytes_saved,omitempty"`   // bytes avoided via RAG
+	TokensSaved  int64     `json:"tokens_saved,omitempty"`  // estimated tokens saved
+	ResponseMs   int64     `json:"response_ms,omitempty"`   // response time in milliseconds
+}
+
+const metricsFile = "search_metrics.jsonl"
+
+// mu protects concurrent writes to the same metrics file.
+var mu sync.Mutex
+
+// AppendSearchMetric appends a single metric line to {workspaceRoot}/.ragcode/search_metrics.jsonl.
+// Thread-safe via mutex. Fails silently (logs nothing) to avoid impacting tool response times.
+func AppendSearchMetric(workspaceRoot string, m SearchMetric) {
+	if workspaceRoot == "" {
+		return
+	}
+
+	m.Timestamp = time.Now()
+
+	line, err := json.Marshal(m)
+	if err != nil {
+		return
+	}
+	line = append(line, '\n')
+
+	dir := filepath.Join(workspaceRoot, ".ragcode")
+	path := filepath.Join(dir, metricsFile)
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	_ = os.MkdirAll(dir, 0o755)
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	_, _ = f.Write(line)
+}

pkg/telemetry/metrics_reader.go

@@ -0,0 +1,66 @@
+package telemetry
+
+import (
+	"bufio"
+	"encoding/json"
+	"os"
+	"path/filepath"
+)
+
+// AggregatedMetrics holds cumulative statistics from search_metrics.jsonl.
+type AggregatedMetrics struct {
+	TotalSearches      int     `json:"total_searches"`
+	SearchesWithResults int    `json:"searches_with_results"`
+	FallbackSearches   int     `json:"fallback_searches"`
+	VectorSearches     int     `json:"vector_searches"`
+	AvgTopScore        float32 `json:"avg_top_score"`
+	TotalBytesSaved    int64   `json:"total_bytes_saved"`
+	TotalTokensSaved   int64   `json:"total_tokens_saved"`
+	AvgResponseMs      int64   `json:"avg_response_ms"`
+}
+
+// ReadAggregatedMetrics reads and aggregates all metrics from the JSONL file.
+// Returns nil if no metrics file exists.
+func ReadAggregatedMetrics(workspaceRoot string) *AggregatedMetrics {
+	if workspaceRoot == "" {
+		return nil
+	}
+	path := filepath.Join(workspaceRoot, ".ragcode", metricsFile)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+
+	var agg AggregatedMetrics
+	var scoreSum float32
+	var msSum int64
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		var m SearchMetric
+		if json.Unmarshal(scanner.Bytes(), &m) != nil {
+			continue
+		}
+		agg.TotalSearches++
+		if m.ResultCount > 0 {
+			agg.SearchesWithResults++
+		}
+		if m.Source == "fallback" {
+			agg.FallbackSearches++
+		} else {
+			agg.VectorSearches++
+		}
+		scoreSum += m.TopScore
+		agg.TotalBytesSaved += m.BytesSaved
+		agg.TotalTokensSaved += m.TokensSaved
+		msSum += m.ResponseMs
+	}
+
+	if agg.TotalSearches == 0 {
+		return nil
+	}
+	agg.AvgTopScore = scoreSum / float32(agg.TotalSearches)
+	agg.AvgResponseMs = msSum / int64(agg.TotalSearches)
+	return &agg
+}

pkg/telemetry/metrics_test.go

@@ -0,0 +1,74 @@
+package telemetry
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAppendAndRead(t *testing.T) {
+	tmp := t.TempDir()
+
+	AppendSearchMetric(tmp, SearchMetric{
+		Tool: "rag_search", Query: "auth", ResultCount: 5,
+		TopScore: 0.85, Source: "vector", BytesSaved: 14000, TokensSaved: 3500, ResponseMs: 120,
+	})
+	AppendSearchMetric(tmp, SearchMetric{
+		Tool: "rag_search", Query: "config", ResultCount: 0,
+		TopScore: 0, Source: "fallback", BytesSaved: 0, TokensSaved: 0, ResponseMs: 80,
+	})
+	AppendSearchMetric(tmp, SearchMetric{
+		Tool: "rag_find_usages", Query: "MyFunc", ResultCount: 3,
+		TopScore: 0.92, Source: "vector", BytesSaved: 5000, TokensSaved: 1250, ResponseMs: 60,
+	})
+
+	// Verify file exists
+	path := filepath.Join(tmp, ".ragcode", metricsFile)
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("metrics file not created: %v", err)
+	}
+
+	agg := ReadAggregatedMetrics(tmp)
+	if agg == nil {
+		t.Fatal("expected aggregated metrics, got nil")
+	}
+
+	if agg.TotalSearches != 3 {
+		t.Errorf("TotalSearches=%d, want 3", agg.TotalSearches)
+	}
+	if agg.SearchesWithResults != 2 {
+		t.Errorf("SearchesWithResults=%d, want 2", agg.SearchesWithResults)
+	}
+	if agg.FallbackSearches != 1 {
+		t.Errorf("FallbackSearches=%d, want 1", agg.FallbackSearches)
+	}
+	if agg.VectorSearches != 2 {
+		t.Errorf("VectorSearches=%d, want 2", agg.VectorSearches)
+	}
+	if agg.TotalBytesSaved != 19000 {
+		t.Errorf("TotalBytesSaved=%d, want 19000", agg.TotalBytesSaved)
+	}
+	if agg.TotalTokensSaved != 4750 {
+		t.Errorf("TotalTokensSaved=%d, want 4750", agg.TotalTokensSaved)
+	}
+}
+
+func TestReadEmptyWorkspace(t *testing.T) {
+	tmp := t.TempDir()
+	agg := ReadAggregatedMetrics(tmp)
+	if agg != nil {
+		t.Error("expected nil for workspace without metrics")
+	}
+}
+
+func TestReadEmptyString(t *testing.T) {
+	agg := ReadAggregatedMetrics("")
+	if agg != nil {
+		t.Error("expected nil for empty workspace")
+	}
+}
+
+func TestAppendEmptyWorkspace(t *testing.T) {
+	// Should not panic
+	AppendSearchMetric("", SearchMetric{Tool: "test"})
+}