feat(status,exec): surface container termination causes (OOMKilled) (#152)

* feat(status,exec): surface container termination causes (OOMKilled)

An OOMKilled container previously showed up as ready=1/2 with reason "-"
in status, and as a bare `container not found` from exec/jobs — every
diagnosis required kubectl and hand-digging containerStatuses.

- PodSummary now carries ContainerIssues (reason/exitCode/finishedAt,
  current-vs-before-restart) extracted from containerStatuses; the pod
  Reason falls back to the termination reason when no waiting reason
  applies.
- `status --details` prints one line per abnormal termination, e.g.
  `container pytorch: OOMKilled (exit 137, finished ...)`, and JSON gains
  a containerIssues array.
- exec FAILED summaries, single-pod exec errors, and jobs list/logs
  errors append the termination cause when the target container is gone,
  via GetContainerRestartInfo (now also reads the current terminated
  state — restartPolicy Never leaves LastTerminationState empty — and
  records FinishedAt).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* style: gofmt podSummaryFromPod literal alignment

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: acmore

docs/command-reference.md

@@ -58,7 +58,8 @@ agents can react without launching a diagnostic chain on every blip:
 - Shows session status for the current or selected session.
 - When `session` is provided, `okdev` can resolve the saved config from session metadata even outside the repo.
 - `--details`: prints a single-session diagnostic view with target selection, pod list, pod IP and node placement, mount persistence, sync path semantics, managed SSH state, key local paths, and target pod details.
-- Detailed JSON output includes per-pod mount metadata and a `pathSemantics` section describing which configured sync paths are shared via the workspace sync and which are expected to survive a session restart.
+- Abnormally terminated containers are surfaced per pod, e.g. `container pytorch: OOMKilled (exit 137, finished 2026-07-05T06:32:11Z)` — both the current state (container stayed down) and the last termination before a restart (suffixed `before last restart`). The pod `reason` column also falls back to the termination reason when no waiting reason applies.
+- Detailed JSON output includes per-pod mount metadata, a `containerIssues` array (`container`, `reason`, `exitCode`, `finishedAt`, `current`), and a `pathSemantics` section describing which configured sync paths are shared via the workspace sync and which are expected to survive a session restart.
 - `--details` is only valid when exactly one session is selected.
 
 ### `okdev target show`
@@ -130,6 +131,7 @@ agents can react without launching a diagnostic chain on every blip:
 - Lists detached `okdev exec --detach` jobs across the session's running pods.
 - Reads job metadata from `/var/okdev/exec/*.json` in the target container (plus the legacy `/tmp/okdev-exec/` location for jobs launched by older okdev versions).
 - If the target container is gone (e.g. OOMKilled), listing and `jobs logs` automatically retry through the `okdev-sidecar` container, which mounts the same runtime volume.
+- When a command fails because the container is gone, the error carries the termination cause when it can be determined, e.g. `container "pytorch" terminated: OOMKilled (exit 137, finished 2026-07-05T06:32:11Z)`. The same hint is appended to `okdev exec` FAILED lines and single-pod exec errors.
 - Text output is grouped by logical job id, with one row per detached launch showing job id, summarized state, pod count, earliest start time, and original command.
 - JSON output includes both the logical job summary and the per-pod `podStates` records.
 - State values: `running` (wrapper alive and user command in flight), `exited` (user command finished; exit code is recorded in `podStates` / JSON), and `orphaned` (metadata still says `running` but the pid has exited or been recycled - typically the wrapper was `SIGKILL`ed or the container was restarted before the completion metadata could be written). Grouped text summaries render forms such as `running(1/2)`, `exited(2/2)`, and `failed(1/2)`.

internal/cli/exec_jobs.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/acmore/okdev/internal/connect"
 	"github.com/acmore/okdev/internal/fanout"
@@ -181,6 +182,11 @@ func collectDetachJobs(ctx context.Context, client connect.ExecClient, namespace
 				}
 			}
 			if err != nil {
+				if isContainerUnavailableExecError(err) {
+					if hint := containerUnavailableHint(ctx, client, namespace, pod.Name, container); hint != "" {
+						err = fmt.Errorf("%w; %s", err, hint)
+					}
+				}
 				results <- podDetachJobsResult{pod: pod.Name, err: err}
 				return
 			}
@@ -361,6 +367,38 @@ func isContainerUnavailableExecError(err error) bool {
 		strings.Contains(msg, "container_exited")
 }
 
+// containerLastStateClient is satisfied by *kube.Client; kept as a narrow
+// interface so exec paths that only hold a connect.ExecClient can probe for
+// the capability.
+type containerLastStateClient interface {
+	GetContainerRestartInfo(ctx context.Context, namespace, pod, container string) (*kube.ContainerRestartInfo, error)
+}
+
+// containerUnavailableHint explains why an exec target container is gone,
+// e.g. `container "pytorch" terminated: OOMKilled (exit 137, finished
+// 2026-07-05T06:32:11Z)`. Returns "" when the cause cannot be determined —
+// callers append it to the raw exec error, never replace it.
+func containerUnavailableHint(ctx context.Context, client any, namespace, pod, container string) string {
+	if strings.TrimSpace(container) == "" {
+		return ""
+	}
+	g, ok := client.(containerLastStateClient)
+	if !ok {
+		return ""
+	}
+	hintCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	ri, err := g.GetContainerRestartInfo(hintCtx, namespace, pod, container)
+	if err != nil || ri == nil || ri.LastReason == "" {
+		return ""
+	}
+	hint := fmt.Sprintf("container %q terminated: %s (exit %d", container, ri.LastReason, ri.LastExitCode)
+	if !ri.LastFinishedAt.IsZero() {
+		hint += ", finished " + ri.LastFinishedAt.UTC().Format(time.RFC3339)
+	}
+	return hint + ")"
+}
+
 func parseDetachMetadataLines(raw string) ([]detachMetadata, error) {
 	lines := strings.Split(raw, "\n")
 	out := make([]detachMetadata, 0)

internal/cli/exec_jobs_test.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/acmore/okdev/internal/kube"
 )
@@ -239,10 +240,17 @@ func TestDetachJobsCommandReconcilesAndReapsTmpFiles(t *testing.T) {
 // fakeContainerExecClient routes exec results by "pod|container" so tests can
 // model a dead target container alongside a healthy sidecar.
 type fakeContainerExecClient struct {
-	mu      sync.Mutex
-	calls   []string
-	outputs map[string]string
-	errs    map[string]error
+	mu          sync.Mutex
+	calls       []string
+	outputs     map[string]string
+	errs        map[string]error
+	restartInfo map[string]*kube.ContainerRestartInfo
+}
+
+func (f *fakeContainerExecClient) GetContainerRestartInfo(ctx context.Context, namespace, pod, container string) (*kube.ContainerRestartInfo, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.restartInfo[pod+"|"+container], nil
 }
 
 func (f *fakeContainerExecClient) ExecInteractive(ctx context.Context, namespace, pod string, tty bool, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
@@ -314,6 +322,52 @@ func TestCollectDetachJobsDoesNotFallBackOnCommandError(t *testing.T) {
 	}
 }
 
+func TestCollectDetachJobsErrorIncludesTerminationHint(t *testing.T) {
+	// When both the target container and the sidecar fallback fail, the pod
+	// error should say WHY the container is gone instead of a bare
+	// "container not found".
+	client := &fakeContainerExecClient{
+		errs: map[string]error{
+			"okdev-sess-worker-0|pytorch":                   errors.New(`container not found ("pytorch")`),
+			"okdev-sess-worker-0|" + syncthingContainerName: errors.New("sidecar unavailable"),
+		},
+		restartInfo: map[string]*kube.ContainerRestartInfo{
+			"okdev-sess-worker-0|pytorch": {
+				LastReason:     "OOMKilled",
+				LastExitCode:   137,
+				LastFinishedAt: time.Date(2026, 7, 5, 6, 32, 11, 0, time.UTC),
+			},
+		},
+	}
+	pods := []kube.PodSummary{{Name: "okdev-sess-worker-0", Phase: "Running"}}
+	_, podErrors := collectDetachJobs(context.Background(), client, "default", pods, "pytorch", "", pdshDefaultFanout)
+	if len(podErrors) != 1 {
+		t.Fatalf("expected one pod error, got %+v", podErrors)
+	}
+	if !strings.Contains(podErrors[0].Error, `container "pytorch" terminated: OOMKilled (exit 137, finished 2026-07-05T06:32:11Z)`) {
+		t.Fatalf("expected termination hint in error, got %q", podErrors[0].Error)
+	}
+}
+
+func TestContainerUnavailableHint(t *testing.T) {
+	client := &fakeContainerExecClient{
+		restartInfo: map[string]*kube.ContainerRestartInfo{
+			"pod-1|pytorch": {LastReason: "OOMKilled", LastExitCode: 137},
+		},
+	}
+	got := containerUnavailableHint(context.Background(), client, "ns", "pod-1", "pytorch")
+	if got != `container "pytorch" terminated: OOMKilled (exit 137)` {
+		t.Fatalf("unexpected hint: %q", got)
+	}
+	if hint := containerUnavailableHint(context.Background(), client, "ns", "pod-1", "other"); hint != "" {
+		t.Fatalf("expected empty hint without termination info, got %q", hint)
+	}
+	// Clients without restart-info capability (plain ExecClient) yield "".
+	if hint := containerUnavailableHint(context.Background(), &fakePdshExecClient{}, "ns", "pod-1", "pytorch"); hint != "" {
+		t.Fatalf("expected empty hint for capability-less client, got %q", hint)
+	}
+}
+
 func TestIsContainerUnavailableExecError(t *testing.T) {
 	cases := []struct {
 		err  error

internal/cli/jobs.go

@@ -583,6 +583,11 @@ func streamDetachJobLog(ctx context.Context, client detachJobClient, namespace s
 		stderr.Reset()
 		err = client.StreamShInContainer(ctx, namespace, row.Pod, syncthingContainerName, script, writer, &stderr)
 	}
+	if err != nil && isContainerUnavailableExecError(err) {
+		if hint := containerUnavailableHint(ctx, client, namespace, row.Pod, row.Container); hint != "" {
+			err = fmt.Errorf("%w; %s", err, hint)
+		}
+	}
 	if err != nil && strings.TrimSpace(stderr.String()) != "" {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))
 	}

internal/cli/pdsh.go

@@ -166,7 +166,13 @@ func newExecCmd(opts *Options) *cobra.Command {
 			if len(execCmd) == 1 && strings.TrimSpace(execCmd[0]) == "" {
 				execCmd = []string{"sh", "-lc", "command -v bash >/dev/null 2>&1 && exec bash || exec sh"}
 			}
-			return runConnectWithClient(cc.kube, cc.namespace, target, execCmd, !noTTY)
+			connectErr := runConnectWithClient(cc.kube, cc.namespace, target, execCmd, !noTTY)
+			if connectErr != nil && isContainerUnavailableExecError(connectErr) {
+				if hint := containerUnavailableHint(cmd.Context(), cc.kube, cc.namespace, target.PodName, target.Container); hint != "" {
+					return fmt.Errorf("%w; %s", connectErr, hint)
+				}
+			}
+			return connectErr
 		},
 	}
 	cmd.Flags().StringVar(&shell, "shell", "", "Shell to start (default auto-detects bash/sh)")
@@ -994,7 +1000,13 @@ func runMultiExec(ctx context.Context, client connect.ExecClient, namespace stri
 	if len(failures) > 0 {
 		parts := make([]string, 0, len(failures))
 		for _, f := range failures {
-			parts = append(parts, formatPodExecFailureSummary(f.pod, container, f.err))
+			part := formatPodExecFailureSummary(f.pod, container, f.err)
+			if isContainerUnavailableExecError(f.err) {
+				if hint := containerUnavailableHint(ctx, client, namespace, f.pod, container); hint != "" {
+					part += " " + hint
+				}
+			}
+			parts = append(parts, part)
 		}
 		fmt.Fprintf(stderr, "\nFAILED:\n%s\n", strings.Join(parts, "\n"))
 		return fmt.Errorf("%d of %d pods failed", len(failures), len(pods))
@@ -1548,7 +1560,13 @@ func runMultiExecScript(ctx context.Context, client scriptCopyClient, namespace
 	if len(failures) > 0 {
 		parts := make([]string, 0, len(failures))
 		for _, f := range failures {
-			parts = append(parts, formatPodExecFailureSummary(f.pod, container, f.err))
+			part := formatPodExecFailureSummary(f.pod, container, f.err)
+			if isContainerUnavailableExecError(f.err) {
+				if hint := containerUnavailableHint(ctx, client, namespace, f.pod, container); hint != "" {
+					part += " " + hint
+				}
+			}
+			parts = append(parts, part)
 		}
 		fmt.Fprintf(stderr, "\nFAILED:\n%s\n", strings.Join(parts, "\n"))
 		return fmt.Errorf("%d of %d pods failed", len(failures), len(pods))

internal/cli/pdsh_test.go

@@ -340,6 +340,33 @@ func TestFilterRunningPods(t *testing.T) {
 	}
 }
 
+func TestRunMultiExecFailureSummaryIncludesTerminationHint(t *testing.T) {
+	// A dead container's FAILED line should carry the termination cause
+	// (OOMKilled + exit + timestamp) instead of a bare kubelet error.
+	client := &fakeContainerExecClient{
+		errs: map[string]error{
+			"worker-0|pytorch": errors.New(`container not found ("pytorch")`),
+		},
+		restartInfo: map[string]*kube.ContainerRestartInfo{
+			"worker-0|pytorch": {
+				LastReason:     "OOMKilled",
+				LastExitCode:   137,
+				LastFinishedAt: time.Date(2026, 7, 5, 6, 32, 11, 0, time.UTC),
+			},
+		},
+	}
+	pods := []kube.PodSummary{{Name: "worker-0", Phase: "Running"}}
+	var stdout, stderr bytes.Buffer
+	sem := make(chan struct{}, 1)
+	err := runMultiExec(context.Background(), client, "default", pods, "pytorch", []string{"true"}, "", 0, true, sem, &stdout, &stderr)
+	if err == nil {
+		t.Fatal("expected failure")
+	}
+	if !strings.Contains(stderr.String(), `container "pytorch" terminated: OOMKilled (exit 137, finished 2026-07-05T06:32:11Z)`) {
+		t.Fatalf("expected termination hint in FAILED summary, got %q", stderr.String())
+	}
+}
+
 func TestNewDetachJobSpecPlacesFilesOnRuntimeVolume(t *testing.T) {
 	// Detached job logs and metadata must prefer the okdev-runtime volume
 	// (shared with the sidecar) so they survive a target-container crash;

internal/cli/status_details.go

@@ -57,18 +57,30 @@ type detailedStatusTarget struct {
 }
 
 type detailedStatusPod struct {
-	Name       string                `json:"name"`
-	Role       string                `json:"role,omitempty"`
-	Attachable bool                  `json:"attachable"`
-	Phase      string                `json:"phase"`
-	Ready      string                `json:"ready"`
-	Restarts   int32                 `json:"restarts"`
-	Reason     string                `json:"reason"`
-	Age        string                `json:"age"`
-	Selected   bool                  `json:"selected"`
-	PodIP      string                `json:"podIP,omitempty"`
-	NodeName   string                `json:"nodeName,omitempty"`
-	Mounts     []detailedStatusMount `json:"mounts,omitempty"`
+	Name            string                         `json:"name"`
+	Role            string                         `json:"role,omitempty"`
+	Attachable      bool                           `json:"attachable"`
+	Phase           string                         `json:"phase"`
+	Ready           string                         `json:"ready"`
+	Restarts        int32                          `json:"restarts"`
+	Reason          string                         `json:"reason"`
+	Age             string                         `json:"age"`
+	Selected        bool                           `json:"selected"`
+	PodIP           string                         `json:"podIP,omitempty"`
+	NodeName        string                         `json:"nodeName,omitempty"`
+	Mounts          []detailedStatusMount          `json:"mounts,omitempty"`
+	ContainerIssues []detailedStatusContainerIssue `json:"containerIssues,omitempty"`
+}
+
+// detailedStatusContainerIssue surfaces an abnormal container termination
+// (OOMKilled, Error, ...) so `ready=1/2` comes with its cause instead of
+// requiring a kubectl round-trip through containerStatuses.
+type detailedStatusContainerIssue struct {
+	Container  string `json:"container"`
+	Reason     string `json:"reason"`
+	ExitCode   int32  `json:"exitCode"`
+	FinishedAt string `json:"finishedAt,omitempty"`
+	Current    bool   `json:"current"`
 }
 
 type detailedStatusMount struct {
@@ -180,18 +192,33 @@ func buildDetailedTarget(view sessionView, cfg *config.DevEnvironment) (detailed
 
 	pods := make([]detailedStatusPod, 0, len(view.Pods))
 	for _, pod := range sortedSessionPods(view.Pods) {
+		issues := make([]detailedStatusContainerIssue, 0, len(pod.ContainerIssues))
+		for _, issue := range pod.ContainerIssues {
+			finishedAt := ""
+			if !issue.FinishedAt.IsZero() {
+				finishedAt = issue.FinishedAt.UTC().Format(time.RFC3339)
+			}
+			issues = append(issues, detailedStatusContainerIssue{
+				Container:  issue.Container,
+				Reason:     issue.Reason,
+				ExitCode:   issue.ExitCode,
+				FinishedAt: finishedAt,
+				Current:    issue.Current,
+			})
+		}
 		pods = append(pods, detailedStatusPod{
-			Name:       pod.Name,
-			Role:       strings.TrimSpace(pod.Labels["okdev.io/workload-role"]),
-			Attachable: !strings.EqualFold(strings.TrimSpace(pod.Labels["okdev.io/attachable"]), "false"),
-			Phase:      pod.Phase,
-			Ready:      pod.Ready,
-			Restarts:   pod.Restarts,
-			Reason:     pod.Reason,
-			Age:        age(pod.CreatedAt),
-			Selected:   pod.Name == view.TargetPod,
-			PodIP:      pod.PodIP,
-			NodeName:   pod.NodeName,
+			Name:            pod.Name,
+			Role:            strings.TrimSpace(pod.Labels["okdev.io/workload-role"]),
+			Attachable:      !strings.EqualFold(strings.TrimSpace(pod.Labels["okdev.io/attachable"]), "false"),
+			Phase:           pod.Phase,
+			Ready:           pod.Ready,
+			Restarts:        pod.Restarts,
+			Reason:          pod.Reason,
+			Age:             age(pod.CreatedAt),
+			Selected:        pod.Name == view.TargetPod,
+			PodIP:           pod.PodIP,
+			NodeName:        pod.NodeName,
+			ContainerIssues: issues,
 		})
 	}
 	return target, pods
@@ -545,6 +572,9 @@ func printDetailedStatus(w io.Writer, detail detailedStatus) {
 				fmt.Fprintf(w, "  %s\n", extra)
 			}
 		}
+		for _, issue := range pod.ContainerIssues {
+			fmt.Fprintf(w, "  %s\n", renderContainerIssueLine(issue))
+		}
 	}
 
 	fmt.Fprintln(w, "\nSSH:")
@@ -665,6 +695,20 @@ func renderMountsLine(mounts []detailedStatusMount) string {
 	return "mounts: " + strings.Join(parts, ", ")
 }
 
+// renderContainerIssueLine formats one abnormal container termination, e.g.
+// "container pytorch: OOMKilled (exit 137, finished 2026-07-05T06:32:11Z)".
+func renderContainerIssueLine(issue detailedStatusContainerIssue) string {
+	line := fmt.Sprintf("container %s: %s (exit %d", issue.Container, issue.Reason, issue.ExitCode)
+	if issue.FinishedAt != "" {
+		line += ", finished " + issue.FinishedAt
+	}
+	line += ")"
+	if !issue.Current {
+		line += " before last restart"
+	}
+	return line
+}
+
 func summarizeConfiguredPorts(forwards []config.PortMapping) []string {
 	lines := make([]string, 0, len(forwards))
 	for _, forward := range forwards {