diff --git a/pkg/agent/ai/analyze/tools/pattern_search.go b/pkg/agent/ai/analyze/tools/pattern_search.go new file mode 100644 index 0000000..e4a7e4a --- /dev/null +++ b/pkg/agent/ai/analyze/tools/pattern_search.go @@ -0,0 +1,227 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/VersusControl/versus-incident/pkg/core" +) + +// PatternSearch browses the agent catalog with filtering and ordering. +// It complements PatternHistory (which requires the exact pattern id) +// by letting the model find candidate patterns from a template +// substring, a verdict, a service, or a rule_name. Useful when the +// model wants to ask "are there other patterns on this service that +// fired recently?" without already knowing their ids. +type PatternSearch struct { + Catalog PatternCatalog +} + +// Name implements core.AnalyzeTool. +func (PatternSearch) Name() string { return "pattern_search" } + +// Description implements core.AnalyzeTool. +func (PatternSearch) Description() string { + return "Search the learned log-pattern catalog. Filter by template substring (case-insensitive), service, verdict, or rule_name; results ordered by count or last_seen. Use it to find candidate patterns when you do not already know an id; for the full record of one known id use pattern_history." +} + +// Recognised values for ArgsSchema enums. Kept private so tests can +// reuse them without exporting from the package surface. +var ( + patternSearchVerdicts = []string{"known", "unknown", "spike"} + patternSearchOrders = []string{"count_desc", "last_seen_desc", "first_seen_desc"} +) + +// ArgsSchema implements core.AnalyzeTool. +func (PatternSearch) ArgsSchema() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "query": map[string]any{ + "type": "string", + "description": "Optional case-insensitive substring matched against the pattern template.", + }, + "service": map[string]any{ + "type": "string", + "description": "Optional service name to filter by (exact, case-insensitive).", + }, + "verdict": map[string]any{ + "type": "string", + "enum": anySliceFromStrings(patternSearchVerdicts), + "description": "Optional verdict filter: known | unknown | spike.", + }, + "rule_name": map[string]any{ + "type": "string", + "description": "Optional named-rule filter (e.g. oom, panic, 5xx-burst), exact match.", + }, + "order_by": map[string]any{ + "type": "string", + "enum": anySliceFromStrings(patternSearchOrders), + "description": "Result ordering. Default count_desc.", + }, + "limit": map[string]any{ + "type": "integer", + "description": "Cap the number of patterns returned. Default 20, max 100.", + }, + }, + } +} + +// anySliceFromStrings widens a []string to []any so it can sit inside +// the loose map[string]any JSON-schema representation Eino accepts. +func anySliceFromStrings(in []string) []any { + out := make([]any, len(in)) + for i, s := range in { + out[i] = s + } + return out +} + +type patternSearchArgs struct { + Query string `json:"query"` + Service string `json:"service"` + Verdict string `json:"verdict"` + RuleName string `json:"rule_name"` + OrderBy string `json:"order_by"` + Limit int `json:"limit"` +} + +type patternSearchItem struct { + ID string `json:"id"` + Template string `json:"template"` + Service string `json:"service,omitempty"` + Source string `json:"source,omitempty"` + RuleName string `json:"rule_name,omitempty"` + Verdict string `json:"verdict,omitempty"` + Tags []string `json:"tags,omitempty"` + Count int `json:"count"` + Baseline float64 `json:"baseline"` + LastSeen string `json:"last_seen"` +} + +// Invoke implements core.AnalyzeTool. +func (p PatternSearch) Invoke(_ context.Context, args json.RawMessage) (*core.ToolResult, error) { + if p.Catalog == nil { + return nil, fmt.Errorf("pattern_search: catalog not configured") + } + var a patternSearchArgs + if len(args) > 0 { + if err := json.Unmarshal(args, &a); err != nil { + return nil, fmt.Errorf("pattern_search: parse args: %w", err) + } + } + + // Defaults + caps. Order matters: clamp BEFORE filtering so the + // limit is honoured against the full filtered set. + if a.Limit <= 0 { + a.Limit = 20 + } + if a.Limit > 100 { + a.Limit = 100 + } + if a.OrderBy == "" { + a.OrderBy = "count_desc" + } + if !containsString(patternSearchOrders, a.OrderBy) { + return nil, fmt.Errorf("pattern_search: invalid order_by %q (want %s)", a.OrderBy, strings.Join(patternSearchOrders, " | ")) + } + if a.Verdict != "" && !containsString(patternSearchVerdicts, strings.ToLower(a.Verdict)) { + return nil, fmt.Errorf("pattern_search: invalid verdict %q (want %s)", a.Verdict, strings.Join(patternSearchVerdicts, " | ")) + } + + queryLower := strings.ToLower(a.Query) + all := p.Catalog.All() + filtered := make([]*PatternView, 0, len(all)) + for _, pat := range all { + if pat == nil { + continue + } + if queryLower != "" && !strings.Contains(strings.ToLower(pat.Template), queryLower) { + continue + } + if a.Service != "" && !strings.EqualFold(pat.Service, a.Service) { + continue + } + if a.Verdict != "" && !strings.EqualFold(pat.Verdict, a.Verdict) { + continue + } + if a.RuleName != "" && pat.RuleName != a.RuleName { + continue + } + filtered = append(filtered, pat) + } + + sortPatternViews(filtered, a.OrderBy) + + totalMatched := len(filtered) + if len(filtered) > a.Limit { + filtered = filtered[:a.Limit] + } + + out := make([]patternSearchItem, 0, len(filtered)) + for _, pat := range filtered { + out = append(out, patternSearchItem{ + ID: pat.ID, + Template: pat.Template, + Service: pat.Service, + Source: pat.Source, + RuleName: pat.RuleName, + Verdict: pat.Verdict, + Tags: pat.Tags, + Count: pat.Count, + Baseline: pat.Baseline, + LastSeen: pat.LastSeen.UTC().Format("2006-01-02T15:04:05Z07:00"), + }) + } + + return &core.ToolResult{ + Tool: PatternSearch{}.Name(), + Found: true, + Data: map[string]any{ + "count": len(out), + "total_matched": totalMatched, + "truncated": totalMatched > len(out), + "order_by": a.OrderBy, + "query": a.Query, + "service": a.Service, + "verdict": a.Verdict, + "rule_name": a.RuleName, + "patterns": out, + }, + }, nil +} + +// sortPatternViews orders the slice in-place by the operator's choice. +// All orderings break ties by id ascending so output is deterministic. +func sortPatternViews(views []*PatternView, orderBy string) { + sort.SliceStable(views, func(i, j int) bool { + a, b := views[i], views[j] + switch orderBy { + case "last_seen_desc": + if !a.LastSeen.Equal(b.LastSeen) { + return a.LastSeen.After(b.LastSeen) + } + case "first_seen_desc": + if !a.FirstSeen.Equal(b.FirstSeen) { + return a.FirstSeen.After(b.FirstSeen) + } + default: // count_desc + if a.Count != b.Count { + return a.Count > b.Count + } + } + return a.ID < b.ID + }) +} + +func containsString(in []string, v string) bool { + for _, s := range in { + if s == v { + return true + } + } + return false +} diff --git a/pkg/agent/ai/analyze/tools/pattern_search_test.go b/pkg/agent/ai/analyze/tools/pattern_search_test.go new file mode 100644 index 0000000..9c62312 --- /dev/null +++ b/pkg/agent/ai/analyze/tools/pattern_search_test.go @@ -0,0 +1,283 @@ +package tools + +import ( + "context" + "testing" + "time" +) + +// fakePatternCatalog is a hand-rolled in-memory PatternCatalog for the +// pattern_search tests. We do NOT depend on pkg/agent here (cycle). +type fakePatternCatalog struct { + patterns map[string]*PatternView +} + +func newFakeCatalog(views ...*PatternView) *fakePatternCatalog { + m := make(map[string]*PatternView, len(views)) + for _, v := range views { + m[v.ID] = v + } + return &fakePatternCatalog{patterns: m} +} + +func (f *fakePatternCatalog) Get(id string) *PatternView { return f.patterns[id] } + +func (f *fakePatternCatalog) All() []*PatternView { + out := make([]*PatternView, 0, len(f.patterns)) + for _, v := range f.patterns { + out = append(out, v) + } + return out +} + +func (f *fakePatternCatalog) AllServices() map[string]ServiceInfo { return nil } + +// TestDefault_PatternSearchRegistration verifies pattern_search is wired +// whenever a catalog is present — alongside pattern_history — and omitted +// when no catalog is configured (same convention as the find_runbook +// registration test in tools_default_test.go). +func TestDefault_PatternSearchRegistration(t *testing.T) { + withCat := Default(nil, newFakeCatalog(), nil, nil, nil, nil, nil, nil, nil) + if !hasTool(withCat, "pattern_search") { + t.Error("pattern_search not registered when catalog is present") + } + if !hasTool(withCat, "pattern_history") { + t.Error("pattern_history missing — registration order broke") + } + + noCat := Default(nil, nil, nil, nil, nil, nil, nil, nil, nil) + if hasTool(noCat, "pattern_search") { + t.Error("pattern_search registered without a catalog") + } +} + +func TestPatternSearch_Metadata(t *testing.T) { + tool := PatternSearch{} + if got := tool.Name(); got != "pattern_search" { + t.Errorf("Name() = %q, want pattern_search", got) + } + if tool.Description() == "" { + t.Error("Description() is empty") + } + schema := tool.ArgsSchema() + if schema["type"] != "object" { + t.Errorf("ArgsSchema type = %v, want object", schema["type"]) + } + props, ok := schema["properties"].(map[string]any) + if !ok { + t.Fatalf("ArgsSchema properties missing") + } + for _, key := range []string{"query", "service", "verdict", "rule_name", "order_by", "limit"} { + if _, ok := props[key]; !ok { + t.Errorf("ArgsSchema missing property %q", key) + } + } +} + +func TestPatternSearch_NilCatalog(t *testing.T) { + if _, err := (PatternSearch{}).Invoke(context.Background(), nil); err == nil { + t.Fatal("expected error when catalog not configured") + } +} + +func TestPatternSearch_BadArgs(t *testing.T) { + tool := PatternSearch{Catalog: newFakeCatalog()} + if _, err := tool.Invoke(context.Background(), []byte("{not json")); err == nil { + t.Fatal("expected error on malformed args") + } +} + +func TestPatternSearch_InvalidEnums(t *testing.T) { + tool := PatternSearch{Catalog: newFakeCatalog()} + if _, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{Verdict: "bogus"})); err == nil { + t.Fatal("expected error for unknown verdict") + } + if _, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{OrderBy: "size_desc"})); err == nil { + t.Fatal("expected error for unknown order_by") + } +} + +func seededCatalog() *fakePatternCatalog { + now := time.Now().UTC() + return newFakeCatalog( + &PatternView{ + ID: "p-conn", Template: "Connection refused on <*>", + Service: "orders", RuleName: "5xx-burst", Verdict: "unknown", + Count: 12, Baseline: 4.2, + FirstSeen: now.Add(-3 * time.Hour), LastSeen: now.Add(-5 * time.Minute), + }, + &PatternView{ + ID: "p-panic", Template: "panic: runtime error <*>", + Service: "orders", RuleName: "panic", Verdict: "spike", + Count: 30, Baseline: 1.5, + FirstSeen: now.Add(-30 * time.Minute), LastSeen: now.Add(-2 * time.Minute), + }, + &PatternView{ + ID: "p-oom", Template: "Out of memory: Killed process <*>", + Service: "billing", RuleName: "oom", Verdict: "known", + Count: 1, Baseline: 1.0, + FirstSeen: now.Add(-24 * time.Hour), LastSeen: now.Add(-1 * time.Hour), + }, + &PatternView{ + ID: "p-info", Template: "user <*> logged in", + Service: "auth", RuleName: "default", Verdict: "known", + Count: 9001, Baseline: 200, + FirstSeen: now.Add(-72 * time.Hour), LastSeen: now, + }, + ) +} + +func TestPatternSearch_QuerySubstringCaseInsensitive(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{Query: "PANIC"})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + out := res.Data["patterns"].([]patternSearchItem) + if len(out) != 1 || out[0].ID != "p-panic" { + t.Errorf("query=PANIC -> %+v, want only p-panic", out) + } +} + +func TestPatternSearch_ServiceAndVerdictFilters(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + // orders + unknown -> only p-conn + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{ + Service: "ORDERS", + Verdict: "unknown", + })) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + out := res.Data["patterns"].([]patternSearchItem) + if len(out) != 1 || out[0].ID != "p-conn" { + t.Errorf("service=orders + verdict=unknown -> %+v, want only p-conn", out) + } +} + +func TestPatternSearch_RuleNameExactMatch(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{RuleName: "oom"})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + out := res.Data["patterns"].([]patternSearchItem) + if len(out) != 1 || out[0].ID != "p-oom" { + t.Errorf("rule_name=oom -> %+v, want only p-oom", out) + } + // rule_name match is case-SENSITIVE on purpose (rule names are operator-authored ids). + res, err = tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{RuleName: "OOM"})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if got := res.Data["count"]; got != 0 { + t.Errorf("rule_name=OOM -> count=%v, want 0 (exact match)", got) + } +} + +func TestPatternSearch_OrderByCountDescDefault(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + out := res.Data["patterns"].([]patternSearchItem) + if len(out) != 4 { + t.Fatalf("len = %d, want 4 (no filters)", len(out)) + } + wantOrder := []string{"p-info", "p-panic", "p-conn", "p-oom"} // counts: 9001 > 30 > 12 > 1 + for i, id := range wantOrder { + if out[i].ID != id { + t.Errorf("order[%d] = %q, want %q", i, out[i].ID, id) + } + } +} + +func TestPatternSearch_OrderByLastSeenDesc(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{OrderBy: "last_seen_desc"})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + out := res.Data["patterns"].([]patternSearchItem) + // last_seen: p-info (now) > p-panic (-2m) > p-conn (-5m) > p-oom (-1h) + wantOrder := []string{"p-info", "p-panic", "p-conn", "p-oom"} + for i, id := range wantOrder { + if out[i].ID != id { + t.Errorf("last_seen order[%d] = %q, want %q", i, out[i].ID, id) + } + } +} + +func TestPatternSearch_LimitTruncation(t *testing.T) { + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{Limit: 2})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if got := res.Data["count"]; got != 2 { + t.Errorf("count = %v, want 2", got) + } + if got := res.Data["total_matched"]; got != 4 { + t.Errorf("total_matched = %v, want 4", got) + } + if got := res.Data["truncated"]; got != true { + t.Errorf("truncated = %v, want true", got) + } +} + +func TestPatternSearch_LimitClamp(t *testing.T) { + // 150 generated patterns -> limit=9999 should clamp to 100. + views := make([]*PatternView, 0, 150) + for i := 0; i < 150; i++ { + views = append(views, &PatternView{ + ID: fakeID(i), + Template: "noise", + Count: 150 - i, // deterministic, distinct counts + }) + } + tool := PatternSearch{Catalog: newFakeCatalog(views...)} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{Limit: 9999})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if got := res.Data["count"]; got != 100 { + t.Errorf("limit 9999 -> count=%v, want clamped to 100", got) + } +} + +func TestPatternSearch_EmptyResultStillFound(t *testing.T) { + // "Found" stays true for list-style tools even with zero matches — + // matches the convention used by recent_incidents / recent_changes. + tool := PatternSearch{Catalog: seededCatalog()} + res, err := tool.Invoke(context.Background(), mustArgs(t, patternSearchArgs{Query: "no-such-substring"})) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if !res.Found { + t.Error("Found = false; list-style tools should keep Found=true with empty list") + } + if got := res.Data["count"]; got != 0 { + t.Errorf("count = %v, want 0", got) + } + if got := res.Data["truncated"]; got != false { + t.Errorf("truncated = %v, want false", got) + } +} + +func fakeID(i int) string { + const digits = "0123456789" + if i == 0 { + return "p-0" + } + out := []byte("p-") + rev := []byte{} + for i > 0 { + rev = append(rev, digits[i%10]) + i /= 10 + } + for j := len(rev) - 1; j >= 0; j-- { + out = append(out, rev[j]) + } + return string(out) +} diff --git a/pkg/agent/ai/analyze/tools/tools.go b/pkg/agent/ai/analyze/tools/tools.go index 9f72237..d7e3de2 100644 --- a/pkg/agent/ai/analyze/tools/tools.go +++ b/pkg/agent/ai/analyze/tools/tools.go @@ -104,12 +104,13 @@ type ServiceExtractor interface { // is unchanged. An empty (but configured) corpus still registers and // returns Found:false rather than an error. func Default(store storage.Provider, cat PatternCatalog, reader SignalReader, redactor LineRedactor, services ServiceExtractor, graph *DependencyGraph, changes ChangeFeed, embedder core.Embedder, runbooks RunbookSearcher) []core.AnalyzeTool { - out := make([]core.AnalyzeTool, 0, 7) + out := make([]core.AnalyzeTool, 0, 8) if store != nil { out = append(out, RecentIncidents{Store: store}) } if cat != nil { out = append(out, PatternHistory{Catalog: cat}) + out = append(out, PatternSearch{Catalog: cat}) out = append(out, DescribeService{Catalog: cat}) } if reader != nil { diff --git a/src/agent/ai-analyze-mode.md b/src/agent/ai-analyze-mode.md index 7994163..a74773e 100644 --- a/src/agent/ai-analyze-mode.md +++ b/src/agent/ai-analyze-mode.md @@ -70,7 +70,7 @@ authentication options, and Docker examples. ## Key Features - **Non-intrusive**: Analyze mode never sends notifications or modifies systems. -- **Read-only tools**: The AI uses tools like `recent_incidents`, `pattern_history`, `describe_service`, `get_related_logs`, `describe_dependencies`, and `recent_changes` to gather context. +- **Read-only tools**: The AI uses tools like `recent_incidents`, `pattern_history`, `pattern_search`, `describe_service`, `get_related_logs`, `describe_dependencies`, and `recent_changes` to gather context. - **Customizable**: Fine-tune the AI's behavior with optional settings. Analyze mode empowers you to make informed decisions by providing structured insights when you need them most. diff --git a/src/agent/analyze-tools/tools.md b/src/agent/analyze-tools/tools.md index 9e41583..098a3d0 100644 --- a/src/agent/analyze-tools/tools.md +++ b/src/agent/analyze-tools/tools.md @@ -32,6 +32,28 @@ service. Use case: *"Is this a brand-new pattern, or a known issue that has spiked above its normal baseline?"* +### `pattern_search` + +Browses the pattern catalog with filters when the AI doesn't already +have a pattern id. Complements `pattern_history` (which is id-only) +by letting the AI look for candidate patterns based on what they look +like. + +- **`query`** — case-insensitive substring matched against the + template (e.g. `"connection refused"`). +- **`service`** — exact, case-insensitive service filter. +- **`verdict`** — `known`, `unknown`, or `spike`. +- **`rule_name`** — exact, case-sensitive named-rule filter + (e.g. `oom`, `panic`, `5xx-burst`). +- **`order_by`** — `count_desc` (default), `last_seen_desc`, or + `first_seen_desc`. +- **`limit`** — default `20`, capped at `100`. The response carries + `total_matched` and `truncated` so the AI knows whether to narrow + the query. + +Use case: *"Are there other patterns on this service that fired +recently?"* or *"Find every spike-verdict pattern in the last day."* + ### `describe_service` Summarises a single service: when it was first seen by the agent and