feat: service management

feat: service management

Author: hoalongnatsu

cmd/main.go

@@ -254,6 +254,17 @@ func startAgent(ctx context.Context, app *fiber.App, cfg c.AgentConfig, gatewayS
 		log.Printf("agent: service_patterns warning: %v", e)
 	}
 
+	// Manual-attribution override store (Service-Override seam). It backs the
+	// service-override admin endpoints AND is installed as the process-wide
+	// resolver so a stored correction re-labels future log signals (and, when
+	// the enterprise metric/trace brains are linked, their signals too). OSS
+	// ships this working store so logs override needs no enterprise module.
+	overrideStore, err := agent.LoadServiceOverrideStore(store)
+	if err != nil {
+		log.Printf("agent: service-override load warning: %v (starting fresh)", err)
+	}
+	agent.SetServiceOverride(overrideStore)
+
 	sources, sourceErrs := agent.BuildSources(cfg)
 	for _, e := range sourceErrs {
 		log.Printf("agent: source warning: %v", e)
@@ -298,7 +309,7 @@ func startAgent(ctx context.Context, app *fiber.App, cfg c.AgentConfig, gatewayS
 		return nil, fmt.Errorf("agent: gateway_secret is not configured — /api/agent/* admin endpoints require a secret")
 	}
 	api := app.Group("/api")
-	controllers.NewAgentController(catalog, shadowLog, detectLog, aiBundle.Runbooks != nil).Register(api)
+	controllers.NewAgentController(catalog, miner, shadowLog, detectLog, overrideStore, aiBundle.Runbooks != nil).Register(api)
 	controllers.NewRunbookAdminController(aiBundle.Runbooks).Register(api)
 
 	return catalog, nil

config/config.yaml

@@ -197,7 +197,7 @@ agent:
 
   catalog:
     persist_interval: 30s
-    auto_promote_after: 50       # in detect mode, this many sightings = "known"
+    auto_promote_after: 100      # in detect mode, this many sightings = "known"
     # Spike detection: a known pattern is re-flagged when its tick-level
     # frequency suddenly exceeds the EWMA (Exponentially Weighted Moving Average) baseline by `spike_multiplier`.
     # Two safety floors keep noise out:
@@ -235,7 +235,7 @@ agent:
     enable: false                   # master switch (env: AGENT_AI_ENABLE)
     # provider selects the model backend (openai | deepseek | qwen | ollama |
     # claude | gemini). Default openai. The api_key below must match the chosen provider.
-    provider: openai
+    provider: openai                # model backend (env: AGENT_AI_PROVIDER)
     api_key: ${AGENT_AI_API_KEY}
     model: "gpt-5.4-mini"
     # temperature: 0.0–2.0 (default 0.2). Set a NEGATIVE value (e.g. -1) to
@@ -273,6 +273,20 @@ agent:
     # Used by: Node.js (Bunyan, Pino, Winston), .NET Serilog (JSON), Java logback-json,
     #          Python python-json-logger, AWS Lambda Powertools, nginx JSON access logs
     - '(?i)"(?:service|svc|app|component)"\s*:\s*"([A-Za-z0-9._-]+)"'
+    # Spring Boot / Logback console (custom pattern that prints the app name where
+    # the level usually goes): "<timestamp>  <service>  [<thread>] ...".
+    #   2026-07-01 05:08:14.502  lead-service  [consumer-0-C-1] WARN k.c.NetworkClient : ...
+    # The service is the first token after the timestamp, anchored by the
+    # bracketed thread that follows it. ANSI colour escapes around the token are
+    # stripped before matching (see ServiceMatcher.Extract). On the vanilla Spring
+    # layout the first token is the LEVEL followed by the PID (not a bracket), so
+    # this rule does not fire and the "---" rule below wins instead.
+    - '^\s*\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,9})?\s+([A-Za-z][A-Za-z0-9._-]*)\s+\['
+    # Bracketed MDC context: "[ <service> , requestID=… , traceID=… , spanID=… ]".
+    # Used by: Java/Logback %X MDC layouts that put the service first in a
+    #          bracketed context block. The trailing request/trace/span id key
+    #          anchors the service so a level token like "[ DEBUG ]" is not caught.
+    - '\[\s*([A-Za-z][A-Za-z0-9._-]*)\s*,\s*(?i:request[_-]?id|trace[_-]?id|span[_-]?id)\b'
     # Two bracketed tokens: [logger] [service] ...
     # Used by: Python Django (`[django.request] [orders] ...`),
     #          some logback layouts that put logger first then MDC service.

pkg/agent/brain_log.go

@@ -92,6 +92,16 @@ func (b *logBrain) Group(ctx context.Context, batch []core.Signal) ([]core.Obser
 			if svc == "" {
 				svc = "_unknown"
 			}
+			// Manual-attribution override (Service-Override seam): an operator's
+			// stored correction WINS over regex detection (and over "_unknown").
+			// The match key is the mined pattern identity or a message substring.
+			// A nil resolver (no override wired) returns svc unchanged.
+			svc = ResolveServiceOverride(ctx, ServiceOverrideInput{
+				SourceType: OverrideSourceLog,
+				Service:    svc,
+				Pattern:    id,
+				Message:    sig.Message,
+			})
 			bk = &bucket{template: template, isNew: isNew, service: svc}
 			buckets[id] = bk
 			order = append(order, id)
@@ -164,11 +174,15 @@ func (b *logBrain) Classify(obs core.Observation, mean, std float64, confident b
 	prevCount := postCount - tickFreq // recover the pre-fold count
 	prevVerdict := p.Verdict          // Upsert never mutates Verdict, so == pre-fold
 
+	// AutoPromoteAfter ≤ 0 disables count-based promotion entirely ("0 disables
+	// the promotion"): a pattern is never marked "known" by sighting count
+	// alone, so it keeps flowing to detect-AI however often it is seen. The 100
+	// default for an UNSET key is supplied by the embedded default_config layer
+	// (loaded as the base before user overrides), so an omitted key arrives here
+	// as 100 — only an explicit 0 (or negative) reaches the disabled branch. A
+	// pattern already promoted to "known" stays known regardless of threshold.
 	threshold := b.cat.AutoPromoteAfter
-	if threshold <= 0 {
-		threshold = 100
-	}
-	isKnown := prevVerdict == "known" || postCount >= threshold
+	isKnown := prevVerdict == "known" || (threshold > 0 && postCount >= threshold)
 	if isKnown {
 		if prevVerdict != "known" {
 			b.catalog.MarkKnown(obs.Key)

pkg/agent/brain_log_test.go

@@ -229,3 +229,135 @@ func TestLogBrain_ClassifyLifecycle(t *testing.T) {
 		t.Errorf("spike baseline = %v, want the pre-fold baseline > 0", v.Baseline)
 	}
 }
+
+// --- auto_promote_after threshold semantics (QA-028) --------------------------
+// Spike is left disabled (SpikeMultiplier == 0) throughout so it can never mask
+// the count-based promotion verdict under test.
+
+// (a) The shipped default (100, supplied by the embedded default_config layer
+// for an unset key) still promotes exactly at the 100th sighting.
+func TestLogBrain_AutoPromoteAfter_DefaultPromotesAt100(t *testing.T) {
+	b, c := newLogBrainForTest(t, config.AgentCatalogConfig{AutoPromoteAfter: 100})
+
+	for i := 0; i < 99; i++ {
+		classifyOnce(t, b, logObs("p", 1))
+	}
+	if got := c.Get("p").Count; got != 99 {
+		t.Fatalf("count = %d, want 99", got)
+	}
+	if c.Get("p").Verdict == "known" {
+		t.Fatalf("promoted early at count=99 (threshold 100)")
+	}
+
+	v := classifyOnce(t, b, logObs("p", 1)) // 100th sighting crosses the threshold
+	if got := c.Get("p").Count; got != 100 {
+		t.Fatalf("count = %d, want 100", got)
+	}
+	if v.Class != core.VerdictKnownPattern {
+		t.Fatalf("at-threshold verdict = %v, want known", v.Class)
+	}
+	if c.Get("p").Verdict != "known" {
+		t.Fatalf("catalog verdict = %q, want known (MarkKnown must fire at 100)", c.Get("p").Verdict)
+	}
+}
+
+// (b) A positive custom threshold promotes exactly at that count, not at 100.
+func TestLogBrain_AutoPromoteAfter_CustomThresholdPromotes(t *testing.T) {
+	b, c := newLogBrainForTest(t, config.AgentCatalogConfig{AutoPromoteAfter: 50})
+
+	for i := 0; i < 49; i++ {
+		classifyOnce(t, b, logObs("p", 1))
+	}
+	if c.Get("p").Verdict == "known" {
+		t.Fatalf("promoted before custom threshold 50 (count=%d)", c.Get("p").Count)
+	}
+
+	v := classifyOnce(t, b, logObs("p", 1)) // 50th sighting
+	if got := c.Get("p").Count; got != 50 {
+		t.Fatalf("count = %d, want 50", got)
+	}
+	if v.Class != core.VerdictKnownPattern {
+		t.Fatalf("at-custom-threshold verdict = %v, want known", v.Class)
+	}
+	if c.Get("p").Verdict != "known" {
+		t.Fatalf("catalog verdict = %q, want known at the custom threshold", c.Get("p").Verdict)
+	}
+}
+
+// (c) QA-028: auto_promote_after: 0 DISABLES count-based promotion — a pattern
+// is never marked "known" no matter how many times it is seen. Drives well past
+// the old 100 fallback to prove the explicit 0 is honoured, not re-mapped.
+func TestLogBrain_AutoPromoteAfter_ZeroDisablesPromotion(t *testing.T) {
+	cat := config.AgentCatalogConfig{
+		AutoPromoteAfter: 0, // documented: disables promotion
+		SpikeMultiplier:  0, // disable spike so it can't mask the verdict
+	}
+	b, c := newLogBrainForTest(t, cat)
+
+	var v core.TypedVerdict
+	for i := 0; i < 12; i++ {
+		v = classifyOnce(t, b, logObs("p", 10)) // 120 sightings, well past 100
+	}
+	if got := c.Get("p").Count; got != 120 {
+		t.Fatalf("count = %d, want 120", got)
+	}
+	if c.Get("p").Verdict == "known" {
+		t.Fatalf("auto_promote_after=0 promoted pattern to %q; 0 must disable promotion (QA-028)", c.Get("p").Verdict)
+	}
+	if v.Class != core.VerdictUnknown {
+		t.Fatalf("verdict at count=120 = %v, want unknown (0 disables promotion)", v.Class)
+	}
+}
+
+// (d) A negative threshold (any value ≤ 0) also disables promotion.
+func TestLogBrain_AutoPromoteAfter_NegativeDisablesPromotion(t *testing.T) {
+	cat := config.AgentCatalogConfig{
+		AutoPromoteAfter: -1,
+		SpikeMultiplier:  0,
+	}
+	b, c := newLogBrainForTest(t, cat)
+
+	var v core.TypedVerdict
+	for i := 0; i < 12; i++ {
+		v = classifyOnce(t, b, logObs("p", 10)) // 120 sightings
+	}
+	if c.Get("p").Verdict == "known" {
+		t.Fatalf("auto_promote_after=-1 promoted pattern to %q; any value ≤0 must disable promotion", c.Get("p").Verdict)
+	}
+	if v.Class != core.VerdictUnknown {
+		t.Fatalf("verdict = %v, want unknown (negative disables promotion)", v.Class)
+	}
+}
+
+// (e) An operator-labelled "known" pattern STAYS known even when count-based
+// promotion is disabled (threshold 0). This bites the `prevVerdict == "known"`
+// clause independently of the count clause: were the guard rewritten as
+// `threshold > 0 && (prevVerdict == "known" || postCount >= threshold)`, a
+// disabled threshold would silently un-suppress a hand-labelled pattern.
+func TestLogBrain_AutoPromoteAfter_AlreadyKnownStaysKnown(t *testing.T) {
+	cat := config.AgentCatalogConfig{
+		AutoPromoteAfter: 0, // count-based promotion disabled
+		SpikeMultiplier:  0, // disable spike so it can't mask the verdict
+	}
+	b, c := newLogBrainForTest(t, cat)
+
+	// Seed the pattern (stays Unknown — 0 disables count-based promotion), then
+	// label it known by hand as an operator would.
+	classifyOnce(t, b, logObs("p", 5))
+	if !c.MarkKnown("p") {
+		t.Fatalf("MarkKnown(p) did not mark the seeded pattern")
+	}
+	if c.Get("p").Verdict != "known" {
+		t.Fatalf("precondition: catalog verdict = %q, want known", c.Get("p").Verdict)
+	}
+
+	// Further sightings must keep it known — the prior "known" verdict wins even
+	// though the count threshold is disabled.
+	v := classifyOnce(t, b, logObs("p", 5))
+	if v.Class != core.VerdictKnownPattern {
+		t.Fatalf("already-known verdict = %v, want known (prior known must win with threshold 0)", v.Class)
+	}
+	if c.Get("p").Verdict != "known" {
+		t.Fatalf("catalog verdict = %q, want known (an already-known pattern must stay known)", c.Get("p").Verdict)
+	}
+}

pkg/agent/catalog.go

@@ -58,8 +58,25 @@ type ServiceInfo struct {
 	// storage.DefaultOrgID ("default"); see Pattern.OrgID.
 	OrgID     string    `json:"org_id,omitempty"`
 	FirstSeen time.Time `json:"first_seen"`
+	// Manual is true when an operator created the service by hand via the admin
+	// API (so it is selectable as an override target BEFORE any signal is
+	// attributed to it). Auto-discovered services leave it false. A manual
+	// service coexists with auto-discovery: RegisterService never clobbers an
+	// existing entry, so a name seen in a real signal keeps its manual flag.
+	Manual bool `json:"manual,omitempty"`
 }
 
+// Sentinel errors for manual-service CRUD, so the admin controller can map them
+// to precise HTTP statuses without string matching.
+var (
+	// ErrServiceExists is returned when creating/renaming to a name that is
+	// already tracked (auto-discovered or manual).
+	ErrServiceExists = fmt.Errorf("service already exists")
+	// ErrServiceNotFound is returned when renaming/deleting a name that is not
+	// tracked.
+	ErrServiceNotFound = fmt.Errorf("service not found")
+)
+
 // Catalog is the in-memory + on-disk pattern store.
 //
 // All public methods are safe for concurrent use. Disk persistence is
@@ -299,6 +316,43 @@ func (c *Catalog) Delete(patternID string) bool {
 	return true
 }
 
+// Reset wipes the entire catalog — every learned pattern AND every discovered
+// service — and persists the empty catalog so training restarts from scratch
+// on the next tick. It returns the number of patterns and services that were
+// removed (the pre-reset view: fleet-wide when a CatalogStore is installed,
+// otherwise this instance's in-memory set).
+//
+// This is the whole-catalog counterpart to Delete/EndServiceGrace: when a
+// CatalogStore is installed the empty state routes through it (so a fleet-wide
+// read view is cleared, not just this instance's working set); otherwise the
+// inline whole-blob path writes an empty "patterns" blob.
+func (c *Catalog) Reset() (patterns int, services int, err error) {
+	// Snapshot the pre-reset counts through the same read view callers see
+	// (store-aware when installed) before clearing.
+	patterns = len(c.All())
+	services = len(c.AllServices())
+
+	c.mu.Lock()
+	c.patterns = make(map[string]*Pattern)
+	c.services = make(map[string]*ServiceInfo)
+	c.dirty = true
+	c.mu.Unlock()
+
+	if s := catalogStore(); s != nil {
+		if err := s.Curate(CatalogEdit{Kind: CatalogEditReset}); err != nil {
+			return patterns, services, err
+		}
+		c.mu.Lock()
+		c.dirty = false
+		c.mu.Unlock()
+		return patterns, services, nil
+	}
+	if err := c.Persist(); err != nil {
+		return patterns, services, err
+	}
+	return patterns, services, nil
+}
+
 // Dirty reports whether there are unflushed changes.
 func (c *Catalog) Dirty() bool {
 	c.mu.RLock()
@@ -443,3 +497,84 @@ func (c *Catalog) RestartServiceGrace(name string) bool {
 	c.dirty = true
 	return true
 }
+
+// ---------------------------------------------------------------------------
+// Manual service CRUD — operator-curated services
+// ---------------------------------------------------------------------------
+
+// Service returns a copy of one tracked service's info and whether it exists.
+// It reads the same unified view AllServices() serves (store-aware when a
+// CatalogStore is installed), so a manual service created on any instance is
+// visible fleet-wide.
+func (c *Catalog) Service(name string) (ServiceInfo, bool) {
+	info, ok := c.AllServices()[name]
+	return info, ok
+}
+
+// CreateService records an operator-created (manual) service so it is
+// selectable as an override target before any signal is attributed to it. The
+// caller (admin controller) validates non-existence first — this is the write.
+// It routes through the CatalogStore when one is installed so the manual
+// service is fleet-visible; otherwise it writes the in-memory + blob path.
+func (c *Catalog) CreateService(name string) error {
+	if s := catalogStore(); s != nil {
+		return s.Curate(CatalogEdit{Kind: CatalogEditCreateService, Service: name})
+	}
+	c.mu.Lock()
+	if _, ok := c.services[name]; ok {
+		c.mu.Unlock()
+		return ErrServiceExists
+	}
+	c.services[name] = &ServiceInfo{OrgID: storage.DefaultOrgID, FirstSeen: time.Now().UTC(), Manual: true}
+	c.dirty = true
+	c.mu.Unlock()
+	// Persist immediately: a manual service has no signal to re-create it and
+	// its override rules persist synchronously, so the two must not diverge.
+	return c.Persist()
+}
+
+// RenameService moves a service entry from oldName to newName, preserving its
+// FirstSeen and manual flag. The caller validates that oldName exists and
+// newName does not. Pattern attribution is not bulk-rewritten (a pattern's
+// Service is a historical label; future signals re-attribute); the admin
+// controller repoints override rules that target the old name so none dangle.
+func (c *Catalog) RenameService(oldName, newName string) error {
+	if s := catalogStore(); s != nil {
+		return s.Curate(CatalogEdit{Kind: CatalogEditRenameService, Service: oldName, NewService: newName})
+	}
+	c.mu.Lock()
+	svc, ok := c.services[oldName]
+	if !ok {
+		c.mu.Unlock()
+		return ErrServiceNotFound
+	}
+	if _, exists := c.services[newName]; exists {
+		c.mu.Unlock()
+		return ErrServiceExists
+	}
+	moved := *svc
+	delete(c.services, oldName)
+	c.services[newName] = &moved
+	c.dirty = true
+	c.mu.Unlock()
+	return c.Persist()
+}
+
+// DeleteService removes a tracked service entry. Returns false when it does not
+// exist. The admin controller gates this on the service being manual and on
+// having no override rules that target it, so a delete never orphans an
+// override.
+func (c *Catalog) DeleteService(name string) bool {
+	if s := catalogStore(); s != nil {
+		return s.Curate(CatalogEdit{Kind: CatalogEditDeleteService, Service: name}) == nil
+	}
+	c.mu.Lock()
+	if _, ok := c.services[name]; !ok {
+		c.mu.Unlock()
+		return false
+	}
+	delete(c.services, name)
+	c.dirty = true
+	c.mu.Unlock()
+	return c.Persist() == nil
+}