Merge pull request #87 from LAA-Software-Engineering/feat/trace-retention-days

leo-aa88 · web-flow · commit 487d2316f3c5 · 2026-04-12T01:21:31.000-03:00
Enforce spec.traces.retentionDays for SQLite traces (lazy GC)
diff --git a/README.md b/README.md
@@ -135,6 +135,7 @@ Notes:
 - **`init`** creates `my-agent-system/` with `apiVersion: agentic.dev/v0` resources and a **`hello`** workflow (native `echo` tool only — **no network**).  
 - **`apply`** in non-interactive environments needs **`--auto-approve`** or **`AGENTCTL_AUTO_APPROVE=1`**.  
 - **`run`** stores traces in the **same** SQLite file used for plan/apply (default **`.agentic/state.db`** under `--project`).  
+- If **`spec.traces.retentionDays`** is a positive integer, runs older than that many **UTC calendar days** (by `runs.started_at`) are deleted lazily on **`run`** and **`logs`** (child trace rows cascade). Unset or non-positive means no pruning.  
 - Use **`logs --run <id>`** after a run if you want a single run’s trace (IDs are printed by **`run`**).  
 
 ### Global flags (common)
diff --git a/internal/cli/logs.go b/internal/cli/logs.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/render"
+	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/spec"
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/state"
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/state/sqlite"
 	"github.com/spf13/cobra"
@@ -76,6 +77,13 @@ func runLogs(cmd *cobra.Command, runID, workflow string) error {
 	}
 	defer func() { _ = st.Close() }()
 
+	if n := spec.TraceRetentionDays(graph); n > 0 {
+		cutoff := time.Now().UTC().AddDate(0, 0, -n)
+		if _, err := st.DeleteRunsStartedBefore(ctx, cutoff); err != nil {
+			return fmt.Errorf("logs: prune trace runs: %w", err)
+		}
+	}
+
 	switch {
 	case runID != "":
 		return writeLogsForRun(cmd, ctx, st, dsn, runID, g)
diff --git a/internal/runtime/local/runner.go b/internal/runtime/local/runner.go
@@ -66,6 +66,13 @@ func (r *Runtime) ExecuteWorkflow(ctx context.Context, opts runtime.WorkflowRunO
 		return "", err
 	}
 
+	if n := spec.TraceRetentionDays(graph); n > 0 {
+		cutoff := r.now().UTC().AddDate(0, 0, -n)
+		if _, err := r.Store.DeleteRunsStartedBefore(ctx, cutoff); err != nil {
+			return "", fmt.Errorf("local: prune trace runs: %w", err)
+		}
+	}
+
 	runID := strings.TrimSpace(opts.RunID)
 	if runID == "" {
 		runID = util.NewRunID()
diff --git a/internal/runtime/local/runner_test.go b/internal/runtime/local/runner_test.go
@@ -2,11 +2,15 @@ package local
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/project"
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/runtime"
+	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/state"
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/state/sqlite"
 	"github.com/LAA-Software-Engineering/agentic-control-plane/internal/trace"
 )
@@ -16,6 +20,11 @@ func testRunProjRoot(t *testing.T) string {
 	return filepath.Join("testdata", "runproj")
 }
 
+func testRetentionProjRoot(t *testing.T) string {
+	t.Helper()
+	return filepath.Join("testdata", "retention")
+}
+
 func TestExecuteWorkflow_persistsRunAndTraceInSQLite(t *testing.T) {
 	ctx := context.Background()
 	st, err := sqlite.Open(ctx, filepath.Join(t.TempDir(), "localrun.db"))
@@ -147,3 +156,47 @@ func TestNewRunID_generatedWhenEmpty(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestExecuteWorkflow_prunesOldTraceRuns(t *testing.T) {
+	ctx := context.Background()
+	st, err := sqlite.Open(ctx, filepath.Join(t.TempDir(), "retention.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+
+	fixed := time.Date(2026, 4, 12, 12, 0, 0, 0, time.UTC)
+	oldID := "stale-run"
+	oldStart := fixed.Add(-72 * time.Hour)
+	if err := st.StartRun(ctx, state.Run{
+		RunID: oldID, WorkflowName: "demo", Env: "local", Status: "succeeded",
+		StartedAt: oldStart, InputJSON: `{}`, TotalCostUSD: 0,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := st.AppendTraceEvent(ctx, oldID, oldStart, trace.EventRunStarted, "", `{}`); err != nil {
+		t.Fatal(err)
+	}
+
+	rt := NewRuntime(testRetentionProjRoot(t), st)
+	rt.Now = func() time.Time { return fixed }
+
+	newID := "fresh-run"
+	_, err = rt.ExecuteWorkflow(ctx, runtime.WorkflowRunOptions{
+		RunID:           newID,
+		WorkflowName:    "demo",
+		EnvironmentName: "staging",
+		Env:             "dev",
+		InputJSON:       []byte(`{"topic":"p"}`),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := st.GetRun(ctx, oldID); !errors.Is(err, sql.ErrNoRows) {
+		t.Fatalf("old run: %v", err)
+	}
+	if _, err := st.GetRun(ctx, newID); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/internal/runtime/local/testdata/retention/agents.yaml b/internal/runtime/local/testdata/retention/agents.yaml
@@ -0,0 +1,9 @@
+apiVersion: agentic.dev/v0
+kind: Agent
+metadata:
+  name: reviewer
+spec:
+  model: mock/gpt-4
+  instructions: Summarize the tool payload as JSON.
+  output:
+    schema: ./schemas/agent-out.schema.json
diff --git a/internal/runtime/local/testdata/retention/environments.yaml b/internal/runtime/local/testdata/retention/environments.yaml
@@ -0,0 +1,10 @@
+apiVersion: agentic.dev/v0
+kind: Environment
+metadata:
+  name: staging
+spec:
+  overrides:
+    agents:
+      reviewer:
+        constraints:
+          timeoutSeconds: 99
diff --git a/internal/runtime/local/testdata/retention/project.yaml b/internal/runtime/local/testdata/retention/project.yaml
@@ -0,0 +1,17 @@
+apiVersion: agentic.dev/v0
+kind: Project
+metadata:
+  name: runproj
+spec:
+  imports:
+    - ./tools.yaml
+    - ./agents.yaml
+    - ./workflows.yaml
+    - ./environments.yaml
+  providers:
+    models:
+      mock:
+        type: mock
+  traces:
+    backend: sqlite
+    retentionDays: 1
diff --git a/internal/runtime/local/testdata/retention/schemas/agent-out.schema.json b/internal/runtime/local/testdata/retention/schemas/agent-out.schema.json
@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "required": ["summary"],
+  "properties": {
+    "summary": { "type": "string" }
+  },
+  "additionalProperties": true
+}
diff --git a/internal/runtime/local/testdata/retention/schemas/in.json b/internal/runtime/local/testdata/retention/schemas/in.json
@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "required": ["topic"],
+  "properties": {
+    "topic": { "type": "string" }
+  },
+  "additionalProperties": true
+}
diff --git a/internal/runtime/local/testdata/retention/tools.yaml b/internal/runtime/local/testdata/retention/tools.yaml
@@ -0,0 +1,6 @@
+apiVersion: agentic.dev/v0
+kind: Tool
+metadata:
+  name: helper
+spec:
+  type: native
diff --git a/internal/runtime/local/testdata/retention/workflows.yaml b/internal/runtime/local/testdata/retention/workflows.yaml
@@ -0,0 +1,21 @@
+apiVersion: agentic.dev/v0
+kind: Workflow
+metadata:
+  name: demo
+spec:
+  input:
+    schema: ./schemas/in.json
+  steps:
+    - id: fetch
+      uses: tool.helper.echo
+      with:
+        topic: "${input.topic}"
+        extra: "x"
+    - id: summarize
+      agent: reviewer
+      with:
+        echo: "${steps.fetch.output.echo}"
+  output:
+    value:
+      topic: "${input.topic}"
+      summary: "${steps.summarize.output.summary}"
diff --git a/internal/runtime/local/testdata/runproj/project.yaml b/internal/runtime/local/testdata/runproj/project.yaml
@@ -12,3 +12,6 @@ spec:
     models:
       mock:
         type: mock
+  traces:
+    backend: sqlite
+    retentionDays: 30
diff --git a/internal/spec/traces.go b/internal/spec/traces.go
@@ -0,0 +1,15 @@
+package spec
+
+// TraceRetentionDays returns spec.traces.retentionDays from the merged project graph, or 0 when
+// unset or non-positive (no pruning; issue #75). Tracing config is project-global (not overridden
+// per Environment in MVP).
+func TraceRetentionDays(g *ProjectGraph) int {
+	if g == nil || g.Spec.Traces == nil {
+		return 0
+	}
+	d := g.Spec.Traces.RetentionDays
+	if d <= 0 {
+		return 0
+	}
+	return d
+}
diff --git a/internal/spec/traces_test.go b/internal/spec/traces_test.go
@@ -0,0 +1,26 @@
+package spec
+
+import "testing"
+
+func TestTraceRetentionDays(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name string
+		g    *ProjectGraph
+		want int
+	}{
+		{"nil graph", nil, 0},
+		{"nil traces", &ProjectGraph{Spec: ProjectSpec{}}, 0},
+		{"zero", &ProjectGraph{Spec: ProjectSpec{Traces: &ProjectTracesConfig{RetentionDays: 0}}}, 0},
+		{"negative", &ProjectGraph{Spec: ProjectSpec{Traces: &ProjectTracesConfig{RetentionDays: -1}}}, 0},
+		{"positive", &ProjectGraph{Spec: ProjectSpec{Traces: &ProjectTracesConfig{RetentionDays: 7}}}, 7},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := TraceRetentionDays(tc.g); got != tc.want {
+				t.Fatalf("TraceRetentionDays = %d want %d", got, tc.want)
+			}
+		})
+	}
+}
diff --git a/internal/state/sqlite/runtime.go b/internal/state/sqlite/runtime.go
@@ -259,3 +259,17 @@ ORDER BY seq ASC
 	}
 	return out, rows.Err()
 }
+
+// DeleteRunsStartedBefore deletes runs older than cutoff (by runs.started_at, RFC3339Nano text compare).
+// Foreign keys cascade to run_steps and trace_events (design doc 14.2, issue #75).
+func (s *Store) DeleteRunsStartedBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+	if s == nil || s.db == nil {
+		return 0, fmt.Errorf("sqlite: nil store")
+	}
+	cut := cutoff.UTC().Format(time.RFC3339Nano)
+	res, err := s.db.ExecContext(ctx, `DELETE FROM runs WHERE started_at < ?`, cut)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
diff --git a/internal/state/sqlite/store_test.go b/internal/state/sqlite/store_test.go
@@ -248,3 +248,62 @@ func TestListRecentRuns_and_ListRunsByWorkflow_order(t *testing.T) {
 		t.Fatalf("ListRunsByWorkflow wf-a = %#v", byA)
 	}
 }
+
+func TestDeleteRunsStartedBefore_cascadesChildRows(t *testing.T) {
+	ctx := context.Background()
+	st, err := Open(ctx, filepath.Join(t.TempDir(), "prune.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+
+	oldStart := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC)
+	newStart := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC)
+	if err := st.StartRun(ctx, state.Run{
+		RunID: "old-run", WorkflowName: "wf", Env: "local", Status: "succeeded",
+		StartedAt: oldStart, InputJSON: `{}`, TotalCostUSD: 0,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.UpsertRunStep(ctx, state.RunStep{
+		RunID: "old-run", StepID: "s1", Status: "done", StartedAt: &oldStart,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := st.AppendTraceEvent(ctx, "old-run", oldStart, "log", "", `{}`); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.StartRun(ctx, state.Run{
+		RunID: "new-run", WorkflowName: "wf", Env: "local", Status: "running",
+		StartedAt: newStart, InputJSON: `{}`, TotalCostUSD: 0,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	cutoff := time.Date(2026, 4, 5, 0, 0, 0, 0, time.UTC)
+	n, err := st.DeleteRunsStartedBefore(ctx, cutoff)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Fatalf("RowsAffected = %d want 1", n)
+	}
+
+	if _, err := st.GetRun(ctx, "old-run"); !errors.Is(err, sql.ErrNoRows) {
+		t.Fatalf("old run: %v", err)
+	}
+	evs, err := st.ListTraceEventsByRunID(ctx, "old-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(evs) != 0 {
+		t.Fatalf("trace events for deleted run: %d", len(evs))
+	}
+	got, err := st.GetRun(ctx, "new-run")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.RunID != "new-run" {
+		t.Fatalf("GetRun new: %+v", got)
+	}
+}
diff --git a/internal/state/store.go b/internal/state/store.go
@@ -40,4 +40,7 @@ type RuntimeStore interface {
 	// ListRunsByWorkflow returns runs for the given workflow_name ordered by started_at descending.
 	ListRunsByWorkflow(ctx context.Context, workflowName string, limit int) ([]Run, error)
 	ListTraceEventsByRunID(ctx context.Context, runID string) ([]TraceEvent, error)
+	// DeleteRunsStartedBefore removes every run with started_at strictly before cutoff (UTC), and
+	// associated run_steps / trace_events (SQLite: ON DELETE CASCADE). Used for trace retention (issue #75).
+	DeleteRunsStartedBefore(ctx context.Context, cutoff time.Time) (deleted int64, err error)
 }
