feat: auto-stop idle sessions and preserve git repo state across restarts (#651)

## Summary

- Add configurable inactivity timeout that automatically stops idle
sessions, reclaiming cluster resources
- Preserve git repo state (local branches, uncommitted/staged changes)
to S3 on pod shutdown and restore on resume, so work is never lost when
pods are stopped or recycled
- Three-tier timeout resolution: session-level `spec.inactivityTimeout`
> project-level `ProjectSettings.spec.inactivityTimeoutSeconds` >
default 24h; set to 0 to disable

  ## What changed

  **CRDs:**
- `AgenticSession`: added `spec.inactivityTimeout`,
`status.lastActivityTime`, `status.stoppedReason`
  - `ProjectSettings`: added `spec.inactivityTimeoutSeconds`

  **Backend:**
- Debounced activity tracking in `agui_proxy.go` updates
`status.lastActivityTime` on AG-UI events (RUN_STARTED,
TEXT_MESSAGE_START, TEXT_MESSAGE_CONTENT, TOOL_CALL_START)
  - `parseStatus()` extracts new status fields for API responses

  **Operator:**
- New `inactivity.go` with `shouldAutoStop()`,
`resolveInactivityTimeout()`, `triggerInactivityStop()`, and
per-namespace ProjectSettings cache
- `monitorPod()` checks inactivity on each tick; re-reads CR before
stopping to avoid races
- `reconciler.go` reads `stop-reason` annotation to set
`status.stoppedReason` and condition reason

  **Frontend:**
  - "Stopped (idle)" badge on session cards
  - Inactivity alert banner in session header (full/actions-only modes)
- "This session was automatically stopped after being idle" message in
hibernated section

  **State-sync:**
- `sync.sh`: on SIGTERM, creates git bundles, uncommitted/staged
patches, and metadata.json per repo
- `hydrate.sh`: restores repos from bundles, checks out saved branch,
applies patches (best-effort)
  - `TerminationGracePeriodSeconds` increased from 30 to 60

  **Tests:**
- `inactivity_test.go`: 16 tests covering `shouldAutoStop`,
`resolveInactivityTimeout`, `triggerInactivityStop`,
`getProjectInactivityTimeout`
  - `agui_proxy_test.go`: 18 subtests for `isActivityEvent`

  ## Test plan

  - [x] Operator and backend unit tests pass (`go test -race ./...`)
  - [x] Frontend builds with zero errors/warnings (`npm run build`)
- [x] Create session with 180s inactivity timeout, verify auto-stop
after idle period
- [x] Verify `status.stoppedReason=inactivity` and "Stopped (idle)"
badge in UI
- [x] Verify git repo with local branch + uncommitted changes survives
stop/resume cycle
  - [x] Verify `inactivityTimeout: 0` disables auto-stop
- [x] Verify sessions without explicit timeout fall back to project
settings, then 24h default

  Fixes: RHOAIENG-49782

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: adalton

components/backend/handlers/sessions.go

@@ -245,6 +245,14 @@ func parseStatus(status map[string]interface{}) *types.AgenticSessionStatus {
 		result.CompletionTime = types.StringPtr(completionTime)
 	}
 
+	if lastActivityTime, ok := status["lastActivityTime"].(string); ok && strings.TrimSpace(lastActivityTime) != "" {
+		result.LastActivityTime = types.StringPtr(lastActivityTime)
+	}
+
+	if stoppedReason, ok := status["stoppedReason"].(string); ok && stoppedReason != "" {
+		result.StoppedReason = types.StringPtr(stoppedReason)
+	}
+
 	// jobName and runnerPodName removed - they go stale on restarts
 	// Use GET /k8s-resources endpoint for live job/pod information
 

components/backend/types/session.go

@@ -18,6 +18,7 @@ type AgenticSessionSpec struct {
 	DisplayName          string             `json:"displayName"`
 	LLMSettings          LLMSettings        `json:"llmSettings"`
 	Timeout              int                `json:"timeout"`
+	InactivityTimeout    *int               `json:"inactivityTimeout,omitempty"`
 	UserContext          *UserContext       `json:"userContext,omitempty"`
 	BotAccount           *BotAccountRef     `json:"botAccount,omitempty"`
 	ResourceOverrides    *ResourceOverrides `json:"resourceOverrides,omitempty"`
@@ -41,6 +42,8 @@ type AgenticSessionStatus struct {
 	Phase              string              `json:"phase,omitempty"`
 	StartTime          *string             `json:"startTime,omitempty"`
 	CompletionTime     *string             `json:"completionTime,omitempty"`
+	LastActivityTime   *string             `json:"lastActivityTime,omitempty"`
+	StoppedReason      *string             `json:"stoppedReason,omitempty"`
 	ReconciledRepos    []ReconciledRepo    `json:"reconciledRepos,omitempty"`
 	ReconciledWorkflow *ReconciledWorkflow `json:"reconciledWorkflow,omitempty"`
 	SDKSessionID       string              `json:"sdkSessionId,omitempty"`

components/backend/websocket/agui_proxy.go

@@ -23,6 +23,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -33,6 +34,30 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+const (
+	// activityDebounceInterval is the minimum interval between CR status updates for lastActivityTime.
+	// Inactivity timeout is measured in hours, so minute-level granularity is sufficient.
+	activityDebounceInterval = 60 * time.Second
+
+	// activityUpdateTimeout bounds how long a single activity status update can take.
+	activityUpdateTimeout = 10 * time.Second
+)
+
+// activityUpdateSem limits concurrent goroutines spawned by updateLastActivityTime.
+// With 60s debounce, at most one goroutine per session is active at a time under normal
+// conditions; this cap protects against pathological bursts (e.g., many sessions starting
+// simultaneously with immediate=true).
+var activityUpdateSem = make(chan struct{}, 50)
+
+// lastActivityUpdateTimes tracks the last time we updated lastActivityTime on the CR
+// for each session to avoid excessive API calls. Key: "namespace/sessionName"
+var lastActivityUpdateTimes sync.Map
+
+// sessionProjectMap maps sessionName → projectName so that persistStreamedEvent
+// (which only receives sessionID) can look up the project for activity tracking.
+// Populated by HandleAGUIRunProxy on each run request.
+var sessionProjectMap sync.Map
+
 // HandleAGUIEvents serves the AG-UI event stream over SSE.  Clients
 // (typically EventSource) connect here to receive all events for a
 // session — both persisted history and live events from active runs.
@@ -180,6 +205,9 @@ func HandleAGUIRunProxy(c *gin.Context) {
 
 	log.Printf("AGUI Proxy: run=%s session=%s/%s msgs=%d", truncID(runID), projectName, sessionName, len(rawMessages))
 
+	// Store project→session mapping for activity tracking in persistStreamedEvent
+	sessionProjectMap.Store(sessionName, projectName)
+
 	// Parse messages for display name generation and hidden metadata
 	var minimalMsgs []types.Message
 	if len(rawMessages) > 0 {
@@ -350,6 +378,16 @@ func persistStreamedEvent(sessionID, runID, threadID, jsonData string) {
 	}
 
 	persistEvent(sessionID, event)
+
+	// Update lastActivityTime on CR for activity events (debounced).
+	// Extract event type to check; projectName is derived from the
+	// sessionID-to-project mapping populated by HandleAGUIRunProxy.
+	eventType, _ := event["type"].(string)
+	if isActivityEvent(eventType) {
+		if projectName, ok := sessionProjectMap.Load(sessionID); ok {
+			updateLastActivityTime(projectName.(string), sessionID, eventType == types.EventTypeRunStarted)
+		}
+	}
 }
 
 // ─── POST /agui/interrupt ────────────────────────────────────────────
@@ -727,3 +765,76 @@ func triggerDisplayNameGenerationIfNeeded(projectName, sessionName string, messa
 	sessionCtx := handlers.ExtractSessionContext(spec)
 	handlers.GenerateDisplayNameAsync(projectName, sessionName, userMessage, sessionCtx)
 }
+
+// isActivityEvent returns true for AG-UI event types that indicate session activity.
+func isActivityEvent(eventType string) bool {
+	switch eventType {
+	case types.EventTypeRunStarted,
+		types.EventTypeTextMessageStart,
+		types.EventTypeTextMessageContent,
+		types.EventTypeToolCallStart:
+		return true
+	default:
+		return false
+	}
+}
+
+// updateLastActivityTime updates the lastActivityTime field on the AgenticSession CR status.
+// Updates are debounced to avoid excessive API calls. RUN_STARTED events bypass the debounce
+// to immediately mark the session as active.
+func updateLastActivityTime(projectName, sessionName string, immediate bool) {
+	if handlers.DynamicClient == nil {
+		log.Printf("Activity tracking: DynamicClient is nil, skipping update for %s/%s", projectName, sessionName)
+		return
+	}
+
+	key := projectName + "/" + sessionName
+	now := time.Now()
+
+	if !immediate {
+		if lastUpdate, ok := lastActivityUpdateTimes.Load(key); ok {
+			if now.Sub(lastUpdate.(time.Time)) < activityDebounceInterval {
+				return // Debounce: too soon since last update
+			}
+		}
+	}
+
+	lastActivityUpdateTimes.Store(key, now)
+
+	// Bound concurrency: drop the update if all slots are busy (debounce will retry later).
+	select {
+	case activityUpdateSem <- struct{}{}:
+	default:
+		return
+	}
+
+	// Run in goroutine to avoid blocking event processing
+	go func() {
+		defer func() { <-activityUpdateSem }()
+
+		gvr := handlers.GetAgenticSessionV1Alpha1Resource()
+		ctx, cancel := context.WithTimeout(context.Background(), activityUpdateTimeout)
+		defer cancel()
+
+		obj, err := handlers.DynamicClient.Resource(gvr).Namespace(projectName).Get(ctx, sessionName, metav1.GetOptions{})
+		if err != nil {
+			log.Printf("Activity tracking: failed to get session %s/%s: %v", projectName, sessionName, err)
+			return
+		}
+
+		status, _, _ := unstructured.NestedMap(obj.Object, "status")
+		if status == nil {
+			status = make(map[string]any)
+		}
+		status["lastActivityTime"] = now.UTC().Format(time.RFC3339)
+		if err := unstructured.SetNestedField(obj.Object, status, "status"); err != nil {
+			log.Printf("Activity tracking: failed to set status for %s/%s: %v", projectName, sessionName, err)
+			return
+		}
+
+		_, err = handlers.DynamicClient.Resource(gvr).Namespace(projectName).UpdateStatus(ctx, obj, metav1.UpdateOptions{})
+		if err != nil {
+			log.Printf("Activity tracking: failed to update lastActivityTime for %s/%s: %v", projectName, sessionName, err)
+		}
+	}()
+}

components/backend/websocket/agui_proxy_test.go

@@ -0,0 +1,55 @@
+package websocket
+
+import (
+	"testing"
+
+	"ambient-code-backend/types"
+)
+
+func TestIsActivityEvent(t *testing.T) {
+	activityEvents := []struct {
+		name      string
+		eventType string
+	}{
+		{"RUN_STARTED", types.EventTypeRunStarted},
+		{"TEXT_MESSAGE_START", types.EventTypeTextMessageStart},
+		{"TEXT_MESSAGE_CONTENT", types.EventTypeTextMessageContent},
+		{"TOOL_CALL_START", types.EventTypeToolCallStart},
+	}
+
+	for _, tc := range activityEvents {
+		t.Run(tc.name+" is activity", func(t *testing.T) {
+			if !isActivityEvent(tc.eventType) {
+				t.Errorf("expected %s to be an activity event", tc.name)
+			}
+		})
+	}
+
+	nonActivityEvents := []struct {
+		name      string
+		eventType string
+	}{
+		{"RUN_FINISHED", types.EventTypeRunFinished},
+		{"RUN_ERROR", types.EventTypeRunError},
+		{"STEP_STARTED", types.EventTypeStepStarted},
+		{"STEP_FINISHED", types.EventTypeStepFinished},
+		{"TEXT_MESSAGE_END", types.EventTypeTextMessageEnd},
+		{"TOOL_CALL_ARGS", types.EventTypeToolCallArgs},
+		{"TOOL_CALL_END", types.EventTypeToolCallEnd},
+		{"STATE_SNAPSHOT", types.EventTypeStateSnapshot},
+		{"STATE_DELTA", types.EventTypeStateDelta},
+		{"MESSAGES_SNAPSHOT", types.EventTypeMessagesSnapshot},
+		{"RAW", types.EventTypeRaw},
+		{"META", types.EventTypeMeta},
+		{"empty string", ""},
+		{"unknown event", "UNKNOWN_EVENT"},
+	}
+
+	for _, tc := range nonActivityEvents {
+		t.Run(tc.name+" is not activity", func(t *testing.T) {
+			if isActivityEvent(tc.eventType) {
+				t.Errorf("expected %s to NOT be an activity event", tc.name)
+			}
+		})
+	}
+}

components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx

@@ -1525,6 +1525,11 @@ export default function ProjectSessionDetailPage({
                         {["Stopped", "Completed", "Failed"].includes(phase) && (
                           <div className="max-w-sm">
                             <h3 className="font-semibold text-lg mb-4">Session Hibernated</h3>
+                            {phase === "Stopped" && session?.status?.stoppedReason === "inactivity" && (
+                              <p className="text-sm text-muted-foreground mb-4">
+                                This session was automatically stopped after being idle. You can resume it to continue working.
+                              </p>
+                            )}
 
                             {/* Session details */}
                             <div className="space-y-3 mb-6 text-left">
@@ -2048,6 +2053,11 @@ export default function ProjectSessionDetailPage({
                             {["Stopped", "Completed", "Failed"].includes(phase) && (
                               <div className="max-w-sm">
                                 <h3 className="font-semibold text-lg mb-4">Session Hibernated</h3>
+                                {phase === "Stopped" && session?.status?.stoppedReason === "inactivity" && (
+                                  <p className="text-sm text-muted-foreground mb-4">
+                                    This session was automatically stopped after being idle. You can resume it to continue working.
+                                  </p>
+                                )}
                                 <Button onClick={handleContinue} size="lg" className="w-full" disabled={continueMutation.isPending}>
                                   {continueMutation.isPending ? (
                                     <>

components/frontend/src/app/projects/[name]/sessions/[sessionName]/session-header.tsx

@@ -2,7 +2,8 @@
 
 import { useState } from 'react';
 import { Button } from '@/components/ui/button';
-import { RefreshCw, Octagon, Trash2, Copy, MoreVertical, Info, Play, Pencil, Download, FileText, Printer, Loader2, HardDrive } from 'lucide-react';
+import { Alert, AlertDescription } from '@/components/ui/alert';
+import { RefreshCw, Octagon, Trash2, Copy, MoreVertical, Info, Play, Pencil, Download, FileText, Printer, Loader2, HardDrive, Clock } from 'lucide-react';
 import { CloneSessionDialog } from '@/components/clone-session-dialog';
 import { SessionDetailsModal } from '@/components/session-details-modal';
 import { EditSessionNameDialog } from '@/components/edit-session-name-dialog';
@@ -51,6 +52,7 @@ export function SessionHeader({
   const canStop = isRunning || phase === "Creating";
   const canResume = phase === "Stopped";
   const canDelete = phase === "Completed" || phase === "Failed" || phase === "Stopped";
+  const stoppedDueToInactivity = phase === "Stopped" && session.status?.stoppedReason === "inactivity";
 
   const { refetch: fetchExportData } = useSessionExport(projectName, session.metadata.name, false);
   const { data: mcpStatus } = useMcpStatus(projectName, session.metadata.name, isRunning);
@@ -270,7 +272,15 @@ export function SessionHeader({
   // Actions only (Stop/Resume buttons) - for below breadcrumb
   if (renderMode === 'actions-only') {
     return (
-      <div>
+      <div className="space-y-2">
+        {stoppedDueToInactivity && (
+          <Alert variant="info">
+            <Clock className="h-4 w-4" />
+            <AlertDescription>
+              This session was automatically stopped after being idle. You can resume it to continue working.
+            </AlertDescription>
+          </Alert>
+        )}
         <div className="flex items-start justify-start">
           <div className="flex gap-2">
             {canStop && (
@@ -305,7 +315,15 @@ export function SessionHeader({
 
   // Full mode (original layout)
   return (
-    <div>
+    <div className="space-y-2">
+      {stoppedDueToInactivity && (
+        <Alert variant="info">
+          <Clock className="h-4 w-4" />
+          <AlertDescription>
+            This session was automatically stopped after being idle. You can resume it to continue working.
+          </AlertDescription>
+        </Alert>
+      )}
       <div className="flex items-start justify-end">
         <div className="flex gap-2">
           <Button

components/frontend/src/components/status-badge.tsx

@@ -120,9 +120,10 @@ export function StatusBadge({
 }
 
 /**
- * Session phase badge with appropriate styling
+ * Session phase badge with appropriate styling.
+ * When stoppedReason is "inactivity", the label changes to "Stopped (idle)".
  */
-export function SessionPhaseBadge({ phase }: { phase: string }) {
+export function SessionPhaseBadge({ phase, stoppedReason }: { phase: string; stoppedReason?: string }) {
   const statusMap: Record<string, StatusVariant> = {
     pending: 'pending',
     creating: 'pending',
@@ -137,7 +138,11 @@ export function SessionPhaseBadge({ phase }: { phase: string }) {
   const status = statusMap[phase.toLowerCase()] || 'default';
   const shouldAnimate = status === 'running' || status === 'stopping';
 
-  return <StatusBadge status={status} label={phase} pulse={shouldAnimate} />;
+  const label = phase === 'Stopped' && stoppedReason === 'inactivity'
+    ? 'Stopped (idle)'
+    : phase;
+
+  return <StatusBadge status={status} label={label} pulse={shouldAnimate} />;
 }
 
 /**

components/frontend/src/components/workspace-sections/sessions-section.tsx

@@ -243,7 +243,7 @@ export function SessionsSection({ projectName }: SessionsSectionProps) {
                           </Link>
                         </TableCell>
                         <TableCell>
-                          <SessionPhaseBadge phase={phase} />
+                          <SessionPhaseBadge phase={phase} stoppedReason={session.status?.stoppedReason} />
                         </TableCell>
                         <TableCell>
                           <span className="text-xs px-2 py-1 rounded border bg-muted/50">

components/frontend/src/types/agentic-session.ts

@@ -23,6 +23,7 @@ export type AgenticSessionSpec = {
 	initialPrompt?: string;
 	llmSettings: LLMSettings;
 	timeout: number;
+	inactivityTimeout?: number;
 	displayName?: string;
 	project?: string;
 	interactive?: boolean;
@@ -159,6 +160,8 @@ export type AgenticSessionStatus = {
 	phase: AgenticSessionPhase;
 	startTime?: string;
 	completionTime?: string;
+	lastActivityTime?: string;
+	stoppedReason?: "user" | "inactivity";
 	reconciledRepos?: ReconciledRepo[];
 	reconciledWorkflow?: ReconciledWorkflow;
 	sdkSessionId?: string;

components/frontend/src/types/api/sessions.ts

@@ -45,6 +45,7 @@ export type AgenticSessionSpec = {
   initialPrompt?: string;
   llmSettings: LLMSettings;
   timeout: number;
+  inactivityTimeout?: number;
   displayName?: string;
   project?: string;
   interactive?: boolean;
@@ -89,6 +90,8 @@ export type AgenticSessionStatus = {
   phase: AgenticSessionPhase;
   startTime?: string;
   completionTime?: string;
+  lastActivityTime?: string;
+  stoppedReason?: "user" | "inactivity";
   jobName?: string;
   runnerPodName?: string;
   reconciledRepos?: ReconciledRepo[];