fix(exec): persist detach logs on runtime volume

Move detached exec logs and metadata to the shared runtime volume so post-mortem job inspection survives target-container crashes, while preserving legacy /tmp compatibility and sidecar read fallback.

Author: acmore

docs/command-reference.md

@@ -110,7 +110,7 @@ agents can react without launching a diagnostic chain on every blip:
 - `--detach`: launches the command in the background and returns immediately. Prints per-pod launch details including job id, pid, log path, and metadata path.
 - When `--log-dir` is used with multiple declared groups, logs are written into per-group subdirectories under the requested path. `--log-dir` has no effect with `--detach` (detached jobs log inside the pod).
 - Detached exec preserves argv to the remote process; it is not flattened back into a single shell string. Detached `--script` uploads a per-pod temp file, launches it, and removes that temp file after completion.
-- Detached jobs store combined stdout/stderr and JSON metadata under `/tmp/okdev-exec/` in the target container. For large outputs (training logs, profiles, dumps), prefer `--log-dir` on the caller or redirect inside your command to a session volume; `/tmp` is typically a small tmpfs and heavy writers can fill it.
+- Detached jobs store combined stdout/stderr and JSON metadata under `/var/okdev/exec/` on the shared okdev-runtime volume, so logs and metadata survive a target-container crash (e.g. OOMKill) and remain readable through the sidecar. If that directory cannot be created or written by the container user (e.g. a user-supplied root-owned volume with a non-root container), the launcher falls back to the legacy `/tmp/okdev-exec/` with a notice on stderr — same lifetime semantics as before. For large outputs (training logs, profiles, dumps), prefer `--log-dir` on the caller or redirect inside your command to a session volume; the runtime volume is a node-local emptyDir and heavy writers can fill it.
 - The `pid` returned from `--detach` is the pid of your command itself (okdev re-parents the launcher so `$!` is your program's pid after `execve`). `kill <pid>` or `okdev exec --pod <pod> -- kill <pid>` therefore targets your command, not a wrapper shell.
 - When using `--detach`, pass the command you want okdev to launch directly. Do not add an extra `nohup ... &` inside the command string.
 - `--timeout`: per-pod command timeout (e.g., `30s`, `5m`). Pods exceeding the timeout are cancelled and reported as failed.
@@ -128,7 +128,8 @@ agents can react without launching a diagnostic chain on every blip:
 ### `okdev jobs list [session]`
 
 - Lists detached `okdev exec --detach` jobs across the session's running pods.
-- Reads job metadata from `/tmp/okdev-exec/*.json` in the target container.
+- Reads job metadata from `/var/okdev/exec/*.json` in the target container (plus the legacy `/tmp/okdev-exec/` location for jobs launched by older okdev versions).
+- If the target container is gone (e.g. OOMKilled), listing and `jobs logs` automatically retry through the `okdev-sidecar` container, which mounts the same runtime volume.
 - Text output is grouped by logical job id, with one row per detached launch showing job id, summarized state, pod count, earliest start time, and original command.
 - JSON output includes both the logical job summary and the per-pod `podStates` records.
 - State values: `running` (wrapper alive and user command in flight), `exited` (user command finished; exit code is recorded in `podStates` / JSON), and `orphaned` (metadata still says `running` but the pid has exited or been recycled - typically the wrapper was `SIGKILL`ed or the container was restarted before the completion metadata could be written). Grouped text summaries render forms such as `running(1/2)`, `exited(2/2)`, and `failed(1/2)`.
@@ -152,6 +153,7 @@ agents can react without launching a diagnostic chain on every blip:
 - `--label`: stream logs from pods matching label selectors.
 - `--exclude`: exclude specific pods from the selected set. Cannot be used with `--pod`.
 - `-f` / `--follow`: keep following until every pod in the job reaches a terminal state.
+- If the job's container is gone (e.g. OOMKilled), logs are read through the `okdev-sidecar` container instead — job logs live on the shared runtime volume and survive the container.
 - If some pod logs are unavailable, okdev still streams the logs it can read and reports the missing pods in a `FAILED:` footer before returning non-zero.
 
 ### `okdev jobs wait <job-id> [session]`

internal/cli/exec_jobs.go

@@ -171,6 +171,15 @@ func collectDetachJobs(ctx context.Context, client connect.ExecClient, namespace
 			defer func() { <-sem }()
 			var stdout bytes.Buffer
 			err := connect.RunOnContainer(ctx, client, namespace, pod.Name, container, detachJobsCommand(), false, nil, &stdout, io.Discard)
+			if err != nil && container != syncthingContainerName && isContainerUnavailableExecError(err) {
+				// The target container is gone (OOMKilled, crashed). Job
+				// metadata lives on the shared /var/okdev volume, so the
+				// always-running sidecar can still read it.
+				stdout.Reset()
+				if sideErr := connect.RunOnContainer(ctx, client, namespace, pod.Name, syncthingContainerName, detachJobsCommand(), false, nil, &stdout, io.Discard); sideErr == nil {
+					err = nil
+				}
+			}
 			if err != nil {
 				results <- podDetachJobsResult{pod: pod.Name, err: err}
 				return
@@ -314,26 +323,44 @@ func detachJobsCommand() []string {
 	// variable (rather than cmdline) is robust against PID reuse even when
 	// the job's cmdline is the arbitrary user command. The find(1) call
 	// also reaps stale .tmp files left behind by killed launchers/wrappers.
-	script := `dir='` + detachMetadataDir + `'
-[ -d "$dir" ] || exit 0
-find "$dir" -maxdepth 1 -name '*.tmp' -type f -mmin +1 -delete 2>/dev/null || true
-for f in "$dir"/*.json; do
-  [ -e "$f" ] || continue
-  contents=$(cat "$f")
-  pid=$(printf '%s' "$contents" | sed -n 's/.*"pid":\([0-9][0-9]*\).*/\1/p' | head -n1)
-  job_id=$(basename "$f" .json)
-  alive=0
-  if [ -n "$pid" ] && [ -r "/proc/$pid/environ" ]; then
-    if tr '\0' '\n' < "/proc/$pid/environ" 2>/dev/null | grep -Fqx "OKDEV_JOB_ID=$job_id"; then
-      alive=1
+	// Both the current metadata dir and the legacy /tmp location are scanned
+	// so jobs launched by an older okdev remain visible after an upgrade.
+	script := `for dir in '` + detachMetadataDir + `' '` + legacyDetachMetadataDir + `'; do
+  [ -d "$dir" ] || continue
+  find "$dir" -maxdepth 1 -name '*.tmp' -type f -mmin +1 -delete 2>/dev/null || true
+  for f in "$dir"/*.json; do
+    [ -e "$f" ] || continue
+    contents=$(cat "$f")
+    pid=$(printf '%s' "$contents" | sed -n 's/.*"pid":\([0-9][0-9]*\).*/\1/p' | head -n1)
+    job_id=$(basename "$f" .json)
+    alive=0
+    if [ -n "$pid" ] && [ -r "/proc/$pid/environ" ]; then
+      if tr '\0' '\n' < "/proc/$pid/environ" 2>/dev/null | grep -Fqx "OKDEV_JOB_ID=$job_id"; then
+        alive=1
+      fi
     fi
-  fi
-  printf '%s\t%s\n' "$alive" "$contents"
+    printf '%s\t%s\n' "$alive" "$contents"
+  done
 done
 `
 	return []string{"sh", "-c", script}
 }
 
+// isContainerUnavailableExecError reports whether an exec failure means the
+// container itself is gone or not running (kubelet/CRI phrasing varies), as
+// opposed to the command failing. Only then is the sidecar fallback useful.
+func isContainerUnavailableExecError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "container not found") ||
+		strings.Contains(msg, "is not running") ||
+		strings.Contains(msg, "not created or running") ||
+		strings.Contains(msg, "stopped container") ||
+		strings.Contains(msg, "container_exited")
+}
+
 func parseDetachMetadataLines(raw string) ([]detachMetadata, error) {
 	lines := strings.Split(raw, "\n")
 	out := make([]detachMetadata, 0)

internal/cli/exec_jobs_test.go

@@ -4,7 +4,10 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
+	"io"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/acmore/okdev/internal/kube"
@@ -222,13 +225,115 @@ func TestDetachJobsCommandReconcilesAndReapsTmpFiles(t *testing.T) {
 		// Best-effort cleanup of stale temp files left by killed wrappers.
 		"-name '*.tmp'",
 		"-mmin +1",
+		// Jobs launched before the /var/okdev move must stay visible, so the
+		// scan covers both the runtime-volume dir and the legacy /tmp dir.
+		"'" + detachMetadataDir + "'",
+		"'" + legacyDetachMetadataDir + "'",
 	} {
 		if !strings.Contains(cmd[2], want) {
 			t.Fatalf("expected detach jobs command to contain %q, got %q", want, cmd[2])
 		}
 	}
 }
 
+// fakeContainerExecClient routes exec results by "pod|container" so tests can
+// model a dead target container alongside a healthy sidecar.
+type fakeContainerExecClient struct {
+	mu      sync.Mutex
+	calls   []string
+	outputs map[string]string
+	errs    map[string]error
+}
+
+func (f *fakeContainerExecClient) ExecInteractive(ctx context.Context, namespace, pod string, tty bool, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+	return f.ExecInteractiveInContainer(ctx, namespace, pod, "", tty, command, stdin, stdout, stderr)
+}
+
+func (f *fakeContainerExecClient) ExecInteractiveInContainer(ctx context.Context, namespace, pod, container string, tty bool, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+	key := pod + "|" + container
+	f.mu.Lock()
+	f.calls = append(f.calls, key)
+	output := f.outputs[key]
+	err := f.errs[key]
+	f.mu.Unlock()
+	if output != "" {
+		fmt.Fprint(stdout, output)
+	}
+	return err
+}
+
+func TestCollectDetachJobsFallsBackToSidecarWhenContainerGone(t *testing.T) {
+	// An OOMKilled target container must not hide job metadata: it lives on
+	// the shared /var/okdev volume, readable via the always-running sidecar.
+	meta := "{\"jobId\":\"job-a\",\"pod\":\"okdev-sess-worker-0\",\"container\":\"pytorch\",\"pid\":100,\"command\":\"python train.py\",\"startedAt\":\"2026-07-05T03:00:00Z\",\"stdoutPath\":\"/var/okdev/exec/job-a.log\",\"stderrPath\":\"/var/okdev/exec/job-a.log\",\"metaPath\":\"/var/okdev/exec/job-a.json\",\"state\":\"running\"}\n"
+	client := &fakeContainerExecClient{
+		outputs: map[string]string{
+			"okdev-sess-worker-0|" + syncthingContainerName: meta,
+		},
+		errs: map[string]error{
+			"okdev-sess-worker-0|pytorch": errors.New(`container not found ("pytorch")`),
+		},
+	}
+	pods := []kube.PodSummary{{Name: "okdev-sess-worker-0", Phase: "Running"}}
+	rows, podErrors := collectDetachJobs(context.Background(), client, "default", pods, "pytorch", "", pdshDefaultFanout)
+	if len(podErrors) != 0 {
+		t.Fatalf("unexpected pod errors: %+v", podErrors)
+	}
+	if len(rows) != 1 || rows[0].JobID != "job-a" {
+		t.Fatalf("expected job-a via sidecar fallback, got %+v", rows)
+	}
+	// The row keeps the job's original container so stop/signal still target
+	// the recorded runtime, not the sidecar.
+	if rows[0].Container != "pytorch" {
+		t.Fatalf("expected row container pytorch, got %q", rows[0].Container)
+	}
+	wantCalls := []string{"okdev-sess-worker-0|pytorch", "okdev-sess-worker-0|" + syncthingContainerName}
+	if len(client.calls) != 2 || client.calls[0] != wantCalls[0] || client.calls[1] != wantCalls[1] {
+		t.Fatalf("expected calls %v, got %v", wantCalls, client.calls)
+	}
+}
+
+func TestCollectDetachJobsDoesNotFallBackOnCommandError(t *testing.T) {
+	// Only container-unavailable errors justify the sidecar retry; a failing
+	// list command must surface as a pod error without a second exec.
+	client := &fakeContainerExecClient{
+		errs: map[string]error{
+			"okdev-sess-worker-0|pytorch": errors.New("command terminated with exit code 2"),
+		},
+	}
+	pods := []kube.PodSummary{{Name: "okdev-sess-worker-0", Phase: "Running"}}
+	rows, podErrors := collectDetachJobs(context.Background(), client, "default", pods, "pytorch", "", pdshDefaultFanout)
+	if len(rows) != 0 {
+		t.Fatalf("expected no rows, got %+v", rows)
+	}
+	if len(podErrors) != 1 || !strings.Contains(podErrors[0].Error, "exit code 2") {
+		t.Fatalf("expected exit-code pod error, got %+v", podErrors)
+	}
+	if len(client.calls) != 1 {
+		t.Fatalf("expected single exec attempt, got %v", client.calls)
+	}
+}
+
+func TestIsContainerUnavailableExecError(t *testing.T) {
+	cases := []struct {
+		err  error
+		want bool
+	}{
+		{nil, false},
+		{errors.New(`container not found ("pytorch")`), true},
+		{errors.New("container pytorch is not running"), true},
+		{errors.New("container is not created or running"), true},
+		{errors.New("cannot exec into a stopped container"), true},
+		{errors.New("command terminated with exit code 1"), false},
+		{errors.New("connection reset by peer"), false},
+	}
+	for _, tc := range cases {
+		if got := isContainerUnavailableExecError(tc.err); got != tc.want {
+			t.Fatalf("isContainerUnavailableExecError(%v) = %v, want %v", tc.err, got, tc.want)
+		}
+	}
+}
+
 func TestCollectDetachJobsFiltersByJobID(t *testing.T) {
 	client := &fakePdshExecClient{
 		outputs: map[string]string{

internal/cli/jobs.go

@@ -574,6 +574,15 @@ func streamDetachJobLog(ctx context.Context, client detachJobClient, namespace s
 	writer := newPrefixedWriter(prefix, out, mu)
 	defer writer.Flush()
 	err := client.StreamShInContainer(ctx, namespace, row.Pod, row.Container, script, writer, &stderr)
+	if err != nil && row.Container != syncthingContainerName && isContainerUnavailableExecError(err) {
+		// The job's container is gone (OOMKilled, crashed). Logs under
+		// /var/okdev live on the shared runtime volume, so read them through
+		// the always-running sidecar instead. Setup failures like these
+		// happen before any output is streamed, so retrying cannot duplicate
+		// log lines.
+		stderr.Reset()
+		err = client.StreamShInContainer(ctx, namespace, row.Pod, syncthingContainerName, script, writer, &stderr)
+	}
 	if err != nil && strings.TrimSpace(stderr.String()) != "" {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))
 	}

internal/cli/jobs_logs_wait_test.go

@@ -101,6 +101,53 @@ func TestRunJobsWaitFailsOnNonZeroExit(t *testing.T) {
 	}
 }
 
+// deadContainerJobsClient wraps fakeJobsClient and fails log streams for the
+// named container, modeling an OOMKilled target while the sidecar stays up.
+type deadContainerJobsClient struct {
+	*fakeJobsClient
+	deadContainer string
+	streamMu      sync.Mutex
+	streamCalls   []string
+}
+
+func (c *deadContainerJobsClient) StreamShInContainer(ctx context.Context, namespace, pod, container, script string, stdout, stderr io.Writer) error {
+	c.streamMu.Lock()
+	c.streamCalls = append(c.streamCalls, pod+"|"+container)
+	c.streamMu.Unlock()
+	if container == c.deadContainer {
+		return fmt.Errorf("container not found (%q)", container)
+	}
+	return c.fakeJobsClient.StreamShInContainer(ctx, namespace, pod, container, script, stdout, stderr)
+}
+
+func TestRunJobsLogsFallsBackToSidecarWhenContainerGone(t *testing.T) {
+	client := &deadContainerJobsClient{
+		fakeJobsClient: &fakeJobsClient{
+			listOutputs: map[string][]string{
+				"okdev-sess-worker-0": {
+					detachMetadataJSON("job-oom", "okdev-sess-worker-0", "pytorch", 100, "orphaned", nil),
+				},
+			},
+			streamPlans: map[string]fakeJobsStreamPlan{
+				"okdev-sess-worker-0|read": {stdout: "last words\n"},
+			},
+		},
+		deadContainer: "pytorch",
+	}
+	pods := []kube.PodSummary{{Name: "okdev-sess-worker-0", Phase: "Running"}}
+	var out bytes.Buffer
+	if err := runJobsLogs(context.Background(), client, "default", pods, "pytorch", "job-oom", pdshDefaultFanout, false, &out); err != nil {
+		t.Fatalf("runJobsLogs: %v", err)
+	}
+	if !strings.Contains(out.String(), "last words") {
+		t.Fatalf("expected log content via sidecar fallback, got %q", out.String())
+	}
+	wantCalls := []string{"okdev-sess-worker-0|pytorch", "okdev-sess-worker-0|" + syncthingContainerName}
+	if len(client.streamCalls) != 2 || client.streamCalls[0] != wantCalls[0] || client.streamCalls[1] != wantCalls[1] {
+		t.Fatalf("expected stream calls %v, got %v", wantCalls, client.streamCalls)
+	}
+}
+
 type fakeJobsClient struct {
 	mu              sync.Mutex
 	listOutputs     map[string][]string

internal/cli/pdsh.go

@@ -25,10 +25,28 @@ import (
 	k8sexec "k8s.io/client-go/util/exec"
 )
 
-const detachMetadataDir = "/tmp/okdev-exec"
+// detachMetadataDir lives on the okdev-runtime emptyDir volume (mounted at
+// /var/okdev in both the target container and the okdev-sidecar), so detached
+// job logs and metadata survive a container crash (e.g. OOMKill) and remain
+// readable through the sidecar. Containers without that mount (a --container
+// override) still work: mkdir -p creates the path on the container overlay,
+// with the same lifetime as the old /tmp location.
+const detachMetadataDir = "/var/okdev/exec"
+
+// legacyDetachMetadataDir is where detached jobs launched by older okdev
+// versions keep their logs and metadata; listing scans it for compatibility.
+const legacyDetachMetadataDir = "/tmp/okdev-exec"
+
 const detachPIDPlaceholder = "__OKDEV_DETACH_PID__"
 const detachExitPlaceholder = "__OKDEV_DETACH_EXIT__"
 
+// detachDirPlaceholder stands in for the metadata directory in job paths and
+// JSON templates until the launcher picks the real directory on the pod:
+// detachMetadataDir when the runtime volume is writable, otherwise the legacy
+// /tmp location (e.g. a user-supplied okdev-runtime volume that a non-root
+// container user cannot write to).
+const detachDirPlaceholder = "__OKDEV_DETACH_DIR__"
+
 func newExecCmd(opts *Options) *cobra.Command {
 	var shell string
 	var scriptPath string
@@ -1150,12 +1168,21 @@ func detachCommand(spec detachJobSpec) []string {
 	wrapper := detachWrapperScript(spec.JobID, spec.Argv, spec.Cleanup, spec.MetaPath, runningTemplate, completionTemplate)
 	script := fmt.Sprintf(
 		"set -eu\n"+
-			"log_path=%s\n"+
-			"meta_path=%s\n"+
-			"wrapper_path=%s\n"+
+			// Prefer the shared runtime volume so logs survive a container
+			// crash; fall back to the legacy /tmp location when the volume is
+			// absent or not writable by the container user (e.g. a
+			// user-supplied okdev-runtime volume with root-only permissions).
+			"dir=%s\n"+
+			"if ! mkdir -p \"$dir\" 2>/dev/null || [ ! -w \"$dir\" ]; then\n"+
+			"  dir=%s\n"+
+			"  echo \"okdev: detach dir not writable, using $dir\" >&2\n"+
+			"  mkdir -p \"$dir\"\n"+
+			"fi\n"+
+			"log_path=\"$dir\"/%s\n"+
+			"meta_path=\"$dir\"/%s\n"+
+			"wrapper_path=\"$dir\"/%s\n"+
 			"launch_json=%s\n"+
-			"mkdir -p %s\n"+
-			"cat >\"$wrapper_path\" <<'OKDEV_DETACH_WRAPPER'\n%s\nOKDEV_DETACH_WRAPPER\n"+
+			"sed \"s|%s|$dir|g\" >\"$wrapper_path\" <<'OKDEV_DETACH_WRAPPER'\n%s\nOKDEV_DETACH_WRAPPER\n"+
 			"chmod 700 \"$wrapper_path\"\n"+
 			"nohup sh \"$wrapper_path\" >\"$log_path\" 2>&1 &\n"+
 			"wrapper_pid=$!\n"+
@@ -1186,13 +1213,16 @@ func detachCommand(spec detachJobSpec) []string {
 			"  echo 'detached job metadata missing pid' >&2\n"+
 			"  exit 1\n"+
 			"fi\n"+
-			"printf '%%s\\n' \"$launch_json\" | sed \"s/%s/$pid/g\"\n",
-		shellutil.Quote(spec.LogPath),
-		shellutil.Quote(spec.MetaPath),
-		shellutil.Quote(spec.ScriptPath),
-		shellutil.Quote(launchTemplate),
+			"printf '%%s\\n' \"$launch_json\" | sed \"s|%s|$dir|g; s/%s/$pid/g\"\n",
 		shellutil.Quote(detachMetadataDir),
+		shellutil.Quote(legacyDetachMetadataDir),
+		shellutil.Quote(filepath.Base(spec.LogPath)),
+		shellutil.Quote(filepath.Base(spec.MetaPath)),
+		shellutil.Quote(filepath.Base(spec.ScriptPath)),
+		shellutil.Quote(launchTemplate),
+		detachDirPlaceholder,
 		wrapper,
+		detachDirPlaceholder,
 		detachPIDPlaceholder,
 	)
 	return []string{"sh", "-c", script}
@@ -1282,9 +1312,12 @@ func newDetachJobSpec(jobID, pod, container, cmd string, argv []string, cleanupP
 		jobID = newDetachJobID()
 	}
 	startedAt := time.Now().UTC().Format(time.RFC3339)
-	logPath := filepath.Join(detachMetadataDir, jobID+".log")
-	metaPath := filepath.Join(detachMetadataDir, jobID+".json")
-	scriptPath := filepath.Join(detachMetadataDir, jobID+".sh")
+	// Paths carry detachDirPlaceholder; the launcher script resolves it to
+	// the real directory on the pod (runtime volume, or legacy /tmp when
+	// that volume is not writable) and seds it into the JSON templates.
+	logPath := detachDirPlaceholder + "/" + jobID + ".log"
+	metaPath := detachDirPlaceholder + "/" + jobID + ".json"
+	scriptPath := detachDirPlaceholder + "/" + jobID + ".sh"
 	return detachJobSpec{
 		JobID:      jobID,
 		Pod:        pod,