Merge pull request #137 from LAA-Software-Engineering/feat/110-trace-redaction

feat(trace): sanitize, redact, and truncate trace payloads (#110)

Author: leo-aa88

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Added
 
+- **Trace payload redaction** (issue #110): trace events are sanitized, key-redacted, and size-capped before SQLite storage. Defaults mask common secret key names; override via `Project.spec.traces.redactKeys`, `maxPayloadBytes`, and `spec.traces.redaction` (`maxDepth`, `maxBytes` for binary previews, `maxStringChars`). HITL edit `argsDiff` is redacted before persistence. Local runs use [trace.NewRecorderForGraph] from project spec.
 - **Optional OpenTelemetry trace export** (issue #108): `Project.spec.telemetry` (`enabled`, `serviceName`, `endpoint` with `env:` tokens, `consoleExport`) emits WayFind-aligned `gen_ai.*` spans (`agent.run`, `model.chat`, `tool.exec`, `approval`) alongside SQLite traces. Disabled by default; init failures log a warning and never fail runs. See [`docs/OTEL.md`](docs/OTEL.md) for a Jaeger quick start.
 - **`agentctl inspect --web`** — read-only local inspector (default `http://127.0.0.1:8787`) over SQLite state: runs, trace timeline, run steps, applied deployment resources, and checkpoints ([#109](https://github.com/LAA-Software-Engineering/agentic-control-plane/issues/109)).
 - **Run checkpointing and resume** (issue #105): SQLite `run_checkpoints` table stores per-run execution snapshots after each completed step. `agentctl run --resume <run-id>` rehydrates interpolation context and continues from the next step without replaying earlier steps. Interrupted runs exit cleanly (status `interrupted`, exit code 0) and cascade with trace retention pruning. Checkpoints are written before step rows are marked succeeded to avoid replay on crash; runs pin `workflow_spec_hash` and `environment_name` for safe resume.

README.md

@@ -149,6 +149,9 @@ Notes:
 - **`Policy.spec.hitl.interruptOn`** keys are **Tool metadata.name** values; they configure review options (edit rules, switch targets) for calls already gated by **`approvals.requiredFor`** or safety metadata — they do not gate tools on their own.  
 - **`run`** stores traces in the **same** SQLite file used for plan/apply (default **`.agentic/state.db`** under `--project`). Optional OTLP export (`spec.telemetry`, off by default) is additive only — see [`docs/OTEL.md`](docs/OTEL.md). When enabled you need `serviceName` plus either `consoleExport: true` or an `endpoint` (`https://…` or `env:VAR`, e.g. `env:OTEL_EXPORTER_OTLP_ENDPOINT`). Export that variable before `run` if you use `env:`; if it is missing or the collector is unreachable, `agentctl` logs a warning, skips OTLP, and the workflow still completes (SQLite traces unchanged).  
 - If **`spec.traces.retentionDays`** is a positive integer, runs older than that many **UTC calendar days** (by `runs.started_at`) are deleted lazily on **`run`** and **`logs`** (child trace rows cascade). Unset or non-positive means no pruning.  
+- **Trace payload redaction** (issue #110): before SQLite storage, event JSON is sanitized, key-redacted, and size-capped. Defaults mask common secret key names (substring match on map keys). Optional project knobs:
+  - **`spec.traces.redactKeys`** / **`maxPayloadBytes`** — merged with defaults; also available under **`spec.traces.redaction`** together with **`maxDepth`**, **`maxStringChars`**, and **`maxBytes`** (max bytes for **binary** previews in sanitized values, not the overall JSON cap).
+  - Stored events may show **`[REDACTED]`**, **`payload_truncated`** / **`preview`**, or depth/binary placeholders in **`logs`** / **`inspect --web`**.
 - Use **`logs --run <id>`** after a run if you want a single run’s trace (IDs are printed by **`run`**).  
 
 ### Global flags (common)

internal/engine/hitl.go

@@ -142,7 +142,12 @@ func (e *Executor) resolvePendingHitl(
 		"resolvedUses": uses,
 	}
 	if decision.Kind == spec.HitlDecisionEdit {
-		traceData["argsDiff"] = policy.HitlArgsDiff(pending.With, with)
+		diff := policy.HitlArgsDiff(pending.With, with)
+		if e.Trace != nil {
+			traceData["argsDiff"] = trace.RedactArgsDiff(diff, gate.Review.RedactKeys, e.Trace.Redaction)
+		} else {
+			traceData["argsDiff"] = diff
+		}
 	}
 	if decision.Kind == spec.HitlDecisionSwitch {
 		traceData["switchTarget"] = decision.SwitchTarget

internal/runtime/local/runner.go

@@ -98,7 +98,7 @@ func (r *Runtime) startWorkflow(ctx context.Context, opts runtime.WorkflowRunOpt
 		return runID, fmt.Errorf("local: start run: %w", err)
 	}
 
-	rec := trace.NewRecorder(r.Store)
+	rec := trace.NewRecorderForGraph(r.Store, prep.graph)
 	if _, err := rec.Append(ctx, runID, "", trace.EventRunStarted, map[string]any{
 		"workflow": wfName, "environment": opts.EnvironmentName,
 	}); err != nil {
@@ -170,7 +170,7 @@ func (r *Runtime) resumeWorkflow(ctx context.Context, opts runtime.WorkflowRunOp
 		return runID, fmt.Errorf("local: mark run running: %w", err)
 	}
 
-	rec := trace.NewRecorder(r.Store)
+	rec := trace.NewRecorderForGraph(r.Store, prep.graph)
 	if _, err := rec.Append(ctx, runID, "", trace.EventRunResumed, map[string]any{
 		"workflow": wfName,
 	}); err != nil {

internal/spec/kinds.go

@@ -41,8 +41,20 @@ type ProjectStateConfig struct {
 }
 
 type ProjectTracesConfig struct {
-	Backend       string `yaml:"backend,omitempty" json:"backend,omitempty"`
-	RetentionDays int    `yaml:"retentionDays,omitempty" json:"retentionDays,omitempty"`
+	Backend         string                     `yaml:"backend,omitempty" json:"backend,omitempty"`
+	RetentionDays   int                        `yaml:"retentionDays,omitempty" json:"retentionDays,omitempty"`
+	RedactKeys      []string                   `yaml:"redactKeys,omitempty" json:"redactKeys,omitempty"`
+	MaxPayloadBytes int                        `yaml:"maxPayloadBytes,omitempty" json:"maxPayloadBytes,omitempty"`
+	Redaction       *ProjectTracesRedactionCfg `yaml:"redaction,omitempty" json:"redaction,omitempty"`
+}
+
+// ProjectTracesRedactionCfg tunes sanitize/redact/truncate for trace payloads (issue #110).
+type ProjectTracesRedactionCfg struct {
+	RedactKeys      []string `yaml:"redactKeys,omitempty" json:"redactKeys,omitempty"`
+	MaxDepth        int      `yaml:"maxDepth,omitempty" json:"maxDepth,omitempty"`
+	MaxBytes        int      `yaml:"maxBytes,omitempty" json:"maxBytes,omitempty"`
+	MaxStringChars  int      `yaml:"maxStringChars,omitempty" json:"maxStringChars,omitempty"`
+	MaxPayloadBytes int      `yaml:"maxPayloadBytes,omitempty" json:"maxPayloadBytes,omitempty"`
 }
 
 // ProjectTelemetryConfig enables optional OpenTelemetry trace export (issue #108).

internal/trace/doc.go

@@ -2,4 +2,7 @@
 //
 // [Recorder] checks that a run row exists before appending (clear failure when StartRun was
 // skipped). Event type strings are defined as Event* constants in events.go (design doc §12.2 I).
+//
+// Before persistence, event payloads pass through sanitize → redact → truncate ([PrepareEventData],
+// issue #110). Use [NewRecorderForGraph] so limits and redactKeys come from Project.spec.traces.
 package trace

internal/trace/recorder.go

@@ -17,13 +17,14 @@ var ErrRunNotFound = errors.New("trace: run not found")
 
 // Recorder appends trace_events rows via [state.RuntimeStore] (design doc §12.2 I, §14.2).
 type Recorder struct {
-	RT    state.RuntimeStore
-	Clock func() time.Time
+	RT        state.RuntimeStore
+	Clock     func() time.Time
+	Redaction RedactionOptions
 }
 
 // NewRecorder returns a recorder backed by rt. rt must not be nil when Append is called.
 func NewRecorder(rt state.RuntimeStore) *Recorder {
-	return &Recorder{RT: rt}
+	return &Recorder{RT: rt, Redaction: NormalizeRedactionOptions(DefaultRedactionOptions())}
 }
 
 func (r *Recorder) now() time.Time {
@@ -57,7 +58,8 @@ func (r *Recorder) Append(ctx context.Context, runID, stepID, typ string, data m
 
 	dataJSON := "{}"
 	if len(data) > 0 {
-		b, err := json.Marshal(data)
+		prepared := PrepareEventData(data, nil, r.Redaction)
+		b, err := json.Marshal(prepared)
 		if err != nil {
 			return 0, fmt.Errorf("trace: marshal event data: %w", err)
 		}

internal/trace/recorder_graph.go

@@ -0,0 +1,14 @@
+package trace
+
+import (
+	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/spec"
+	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/state"
+)
+
+// NewRecorderForGraph returns a recorder with redaction options from project spec.
+func NewRecorderForGraph(rt state.RuntimeStore, g *spec.ProjectGraph) *Recorder {
+	return &Recorder{
+		RT:        rt,
+		Redaction: NormalizeRedactionOptions(RedactionFromGraph(g)),
+	}
+}