feat(sync,exec): okdev sync wait; split command-exit from delivery-failure exit semantics (#159)

* feat(sync,exec): add okdev sync wait; split command-exit from delivery-failure semantics

okdev sync wait [session] [--timeout 10m] blocks until every sync mapping
has zero pending bytes in both directions, giving scripts the edit-run
guarantee (vim f.py && okdev sync wait && okdev exec -- python f.py). It
forces a rescan on both sides first so files written moments earlier are
indexed immediately instead of after the FS-watcher delay, and fails fast
with guidance when background sync is not running.

Fanout exec failures are now layered: when every failing pod merely
reported a non-zero command exit, the summary is COMMAND EXITED NON-ZERO
and okdev exits with the remote status (single pod, preserving the
documented verbatim-exit contract) or 1 (multi-pod). Any delivery failure
(unreachable, timeout, container gone) keeps the FAILED framing and maps
to new dedicated exit code 69 (sysexits EX_UNAVAILABLE) via the
ErrExecInfraFailure sentinel, so scripts can tell "my command failed"
from "okdev could not deliver the command" without parsing output.

Covered by unit tests plus kind e2e scenarios (sync wait convergence
guarantee, single-pod exit-status preservation, multi-pod non-zero
framing). Docs and skill references updated.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(e2e): write sync-wait probe file inside the sync root

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: acmore

docs/command-reference.md

@@ -17,10 +17,11 @@ Commands that resolve a session distinguish failure classes so scripts and
 agents can react without launching a diagnostic chain on every blip:
 
 - `0`: success.
-- non-zero from a remote command (e.g. `okdev exec -- false` → `1`, `exit 7` → `7`): the remote process's own status, always preserved.
+- non-zero from a remote command (e.g. `okdev exec -- false` → `1`, `exit 7` → `7`): the remote process's own status, preserved verbatim on single-pod runs. A multi-pod fanout where the command exited non-zero on some pods exits `1` (there is no single status to preserve).
 - `74`: the cluster was reachable but the session has no pods — it is genuinely gone.
 - `78`: a transient cluster-contact failure (API timeout, connection refused/reset, server overload). okdev already retries once with backoff before surfacing this; on `78` the caller should retry rather than treat the session as dead.
-- `1`: any other error (configuration, permissions/RBAC, a failed job result, etc.).
+- `69`: a fanout `okdev exec` could not run the command on at least one pod (unreachable, timeout, container gone) — a delivery failure, as opposed to the command itself exiting non-zero, which stays exit `1`.
+- `1`: any other error (configuration, permissions/RBAC, a failed job result, a fanout command that exited non-zero on some pods, etc.).
 
 ## Commands
 
@@ -50,6 +51,7 @@ agents can react without launching a diagnostic chain on every blip:
 - `okdev ports`
 - `okdev port-forward [session] <local:remote>... [--address <addr>[,<addr>...]] [--pod <name> | --role <role>] [--ready-only]`
 - `okdev sync [--mode up|down|bi] [--foreground] [--reset] [--dry-run]`
+- `okdev sync wait [session] [--timeout 10m]`
 - `okdev prune [--ttl-hours 72] [--all-namespaces] [--all-users] [--include-pvc] [--dry-run]`
 - `okdev migrate [--template <name>] [--set key=value] [--dry-run] [--yes]`
 - `okdev upgrade`
@@ -98,6 +100,7 @@ agents can react without launching a diagnostic chain on every blip:
 - `--parallel` runs declared groups in parallel. `--fanout` caps total concurrent pod executions across all groups.
 - `--script <file>` uploads a local script file and runs it across the targeted pods. In v1 this is local-file-only; `--script -` is not supported.
 - With `--script`, everything after `--` is passed to the script as positional arguments (`$1`, `$2`, … / `"$@"`): `okdev exec --script ./bench.sh -- --steps 100 --warmup 10` runs `bench.sh` on each pod with `$1=--steps $2=100 $3=--warmup $4=10`. Arguments containing spaces are preserved.
+- Failure framing for `-- command...` fanout: when every failing pod merely reported a non-zero exit code from the command itself, the summary is headed `COMMAND EXITED NON-ZERO:` and `okdev` exits with the remote status on a single-pod run (`exit 3` → `3`) or `1` on a multi-pod run — either way `okdev exec -- cmd && next` still short-circuits `&&` chains. When any pod could not run the command at all (unreachable, timeout, container gone), the summary is headed `FAILED:` and `okdev` exits `69`, letting scripts tell "my command failed" apart from "delivery failed" without parsing output.
 - The `-- command...` path is non-interactive fanout execution, not a terminal session. TTY-dependent programs such as `watch`, `top`, `htop`, or full-screen TUIs are not expected to work there; use `okdev ssh` or `okdev exec` without `-- command...` to open an interactive shell first.
 - `-- command...` preserves raw argv to the remote process. If you choose `bash -lc '...'`, that remote shell layer is still yours.
 - `--script` avoids the quoting/stitching problem for non-trivial shell pipelines by uploading the file and running it remotely instead of embedding the body in a one-liner.
@@ -326,6 +329,13 @@ agents can react without launching a diagnostic chain on every blip:
 - `--reset --force --local` / `--reset --force --mesh`: force reset a specific component.
 - `okdev init` writes the starter config and, for built-in templates, a starter local `.stignore` file for the initialized sync root.
 
+### `okdev sync wait [session] [--timeout 10m]`
+
+- Blocks until every configured sync mapping has zero pending bytes in **both** directions, then returns — the edit-run loop guarantee: `vim train.py && okdev sync wait && okdev exec -- python train.py`.
+- Triggers an immediate rescan on both sides before waiting, so files written moments earlier are picked up now instead of after the filesystem-watcher delay.
+- Purely a wait: it does not start or repair sync. If background sync is not running it fails fast with guidance (`okdev sync` starts it, `okdev sync --reset` repairs it).
+- Prints pending-byte progress while waiting; exits non-zero if convergence is not reached within `--timeout`.
+
 ### `okdev upgrade`
 
 - Checks the latest GitHub release and upgrades the `okdev` binary in place.

internal/cli/cluster_errors.go

@@ -21,6 +21,11 @@ const (
 	// ExitSessionNotFound signals that the cluster was reachable but the
 	// requested session has no pods — a real "it's gone" condition.
 	ExitSessionNotFound = 74
+	// ExitExecInfraFailure signals that a fanout exec could not run the
+	// command on at least one pod (unreachable, timeout, container gone) —
+	// as opposed to the command itself exiting non-zero, which stays exit
+	// 1. Follows sysexits EX_UNAVAILABLE.
+	ExitExecInfraFailure = 69
 )
 
 // ErrSessionNotFound is returned when session resolution reaches the cluster
@@ -33,6 +38,13 @@ var ErrSessionNotFound = errors.New("session not found")
 // session that no longer exists.
 var ErrTransientCluster = errors.New("transient cluster contact failure")
 
+// ErrExecInfraFailure marks a fanout exec where at least one pod could not
+// run the command at all (transport, timeout, container gone). Wrapped into
+// the returned error so the top-level classifier maps it to
+// ExitExecInfraFailure, letting scripts tell real delivery failures apart
+// from a command that simply exited non-zero (plain exit 1).
+var ErrExecInfraFailure = errors.New("exec delivery failure")
+
 // ClassifiedExitCode maps okdev's sentinel cluster errors to their dedicated
 // exit codes. It returns ok=false for anything it does not recognize so the
 // caller can fall back to its default (e.g. a remote command's own exit code,
@@ -44,6 +56,8 @@ func ClassifiedExitCode(err error) (int, bool) {
 		return ExitSessionNotFound, true
 	case errors.Is(err, ErrTransientCluster):
 		return ExitTransientCluster, true
+	case errors.Is(err, ErrExecInfraFailure):
+		return ExitExecInfraFailure, true
 	default:
 		return 0, false
 	}

internal/cli/pdsh.go

@@ -1003,20 +1003,46 @@ func runMultiExec(ctx context.Context, client connect.ExecClient, namespace stri
 		}
 	}
 	if len(failures) > 0 {
-		parts := make([]string, 0, len(failures))
-		for _, f := range failures {
-			part := formatPodExecFailureSummary(f.pod, container, f.err)
-			if isContainerUnavailableExecError(f.err) {
-				if hint := containerUnavailableHint(ctx, client, namespace, f.pod, container); hint != "" {
-					part += " " + hint
-				}
+		return reportFanoutFailures(ctx, client, namespace, container, failures, len(pods), stderr)
+	}
+	return nil
+}
+
+// reportFanoutFailures prints the per-pod failure summary and builds the
+// command error. When every failure is the remote command merely exiting
+// non-zero (grep with no match, a failing test), the run is reported as a
+// command result — no "pods failed" framing; a single-pod run preserves the
+// remote exit status verbatim, a multi-pod run exits 1. Any delivery
+// failure (transport, timeout, container gone) keeps the FAILED framing and
+// wraps ErrExecInfraFailure so okdev exits with the dedicated infra code
+// (ExitExecInfraFailure), letting scripts and monitors tell the two apart.
+func reportFanoutFailures(ctx context.Context, client connect.ExecClient, namespace, container string, failures []podExecResult, total int, stderr io.Writer) error {
+	remoteExitOnly := true
+	parts := make([]string, 0, len(failures))
+	for _, f := range failures {
+		if kind, _ := classifyPodExecFailure(f.err); kind != "remote-exit" {
+			remoteExitOnly = false
+		}
+		part := formatPodExecFailureSummary(f.pod, container, f.err)
+		if isContainerUnavailableExecError(f.err) {
+			if hint := containerUnavailableHint(ctx, client, namespace, f.pod, container); hint != "" {
+				part += " " + hint
 			}
-			parts = append(parts, part)
 		}
-		fmt.Fprintf(stderr, "\nFAILED:\n%s\n", strings.Join(parts, "\n"))
-		return fmt.Errorf("%d of %d pods failed", len(failures), len(pods))
+		parts = append(parts, part)
+	}
+	if remoteExitOnly {
+		fmt.Fprintf(stderr, "\nCOMMAND EXITED NON-ZERO:\n%s\n", strings.Join(parts, "\n"))
+		if total == 1 {
+			// Single-pod runs keep the documented "remote exit status is
+			// always preserved" contract: wrap the pod's own ExitError so
+			// the top-level exit-code mapper propagates it verbatim.
+			return fmt.Errorf("command exited non-zero: %w", failures[0].err)
+		}
+		return fmt.Errorf("command exited non-zero on %d of %d pod(s)", len(failures), total)
 	}
-	return nil
+	fmt.Fprintf(stderr, "\nFAILED:\n%s\n", strings.Join(parts, "\n"))
+	return fmt.Errorf("%d of %d pods failed: %w", len(failures), total, ErrExecInfraFailure)
 }
 
 func formatPodExecFailureSummary(podName, container string, err error) string {
@@ -1588,18 +1614,7 @@ func runMultiExecScript(ctx context.Context, client scriptCopyClient, namespace
 		}
 	}
 	if len(failures) > 0 {
-		parts := make([]string, 0, len(failures))
-		for _, f := range failures {
-			part := formatPodExecFailureSummary(f.pod, container, f.err)
-			if isContainerUnavailableExecError(f.err) {
-				if hint := containerUnavailableHint(ctx, client, namespace, f.pod, container); hint != "" {
-					part += " " + hint
-				}
-			}
-			parts = append(parts, part)
-		}
-		fmt.Fprintf(stderr, "\nFAILED:\n%s\n", strings.Join(parts, "\n"))
-		return fmt.Errorf("%d of %d pods failed", len(failures), len(pods))
+		return reportFanoutFailures(ctx, client, namespace, container, failures, len(pods), stderr)
 	}
 	return nil
 }

internal/cli/pdsh_fanout_test.go

@@ -307,7 +307,7 @@ func TestRunExecPodGroupsSurfacesGroupErrors(t *testing.T) {
 		Stdout:     &stdout,
 		Stderr:     &stderr,
 	})
-	if err == nil || err.Error() != "1 of 1 pods failed" {
+	if err == nil || err.Error() != "1 of 1 pods failed: exec delivery failure" {
 		t.Fatalf("expected verbatim single-group error, got %v", err)
 	}
 }

internal/cli/pdsh_test.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/acmore/okdev/internal/kube"
 	k8sexec "k8s.io/client-go/util/exec"
+	utilexec "k8s.io/utils/exec"
 )
 
 func TestFilterPodsByRole(t *testing.T) {
@@ -711,15 +712,50 @@ func TestRunMultiExecPartialFailure(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected error for partial failure")
 	}
+	// A pure remote non-zero exit is a command result, not an okdev
+	// failure: softer framing, plain error (exit 1), no infra sentinel.
+	if !strings.Contains(err.Error(), "command exited non-zero on 1 of 2 pod(s)") {
+		t.Fatalf("unexpected error message: %v", err)
+	}
+	if errors.Is(err, ErrExecInfraFailure) {
+		t.Fatalf("remote-exit-only run must not carry the infra sentinel: %v", err)
+	}
 	got := stderr.String()
-	if !strings.Contains(got, "\nFAILED:\n") {
-		t.Fatalf("expected blank line before multiline failure header, got %q", got)
+	if !strings.Contains(got, "\nCOMMAND EXITED NON-ZERO:\n") || strings.Contains(got, "FAILED:") {
+		t.Fatalf("expected command-result framing without FAILED header, got %q", got)
 	}
 	if !strings.Contains(got, "pod=okdev-sess-worker-1 container=dev kind=remote-exit exit=1") {
 		t.Fatalf("expected classified failure summary for worker-1, got %q", got)
 	}
 }
 
+func TestRunMultiExecSinglePodPreservesRemoteExitStatus(t *testing.T) {
+	client := &fakePdshExecClient{
+		outputs: map[string]string{},
+		errs: map[string]error{
+			"okdev-sess-master-0": k8sexec.CodeExitError{Err: errors.New("command terminated with exit code 7"), Code: 7},
+		},
+	}
+	pods := []kube.PodSummary{{Name: "okdev-sess-master-0", Phase: "Running"}}
+	var stdout, stderr bytes.Buffer
+	err := runMultiExec(context.Background(), client, "default", pods, "dev", []string{"sh", "-lc", "exit 7"}, "", 0, false, make(chan struct{}, pdshDefaultFanout), &stdout, &stderr)
+	if err == nil {
+		t.Fatal("expected error for non-zero remote exit")
+	}
+	// The documented contract: a single-pod remote exit status is preserved
+	// verbatim, so the wrapped ExitError must survive for main's exit mapper.
+	var exitErr utilexec.ExitError
+	if !errors.As(err, &exitErr) || exitErr.ExitStatus() != 7 {
+		t.Fatalf("expected wrapped ExitError with status 7, got %v", err)
+	}
+	if errors.Is(err, ErrExecInfraFailure) {
+		t.Fatalf("remote-exit-only run must not carry the infra sentinel: %v", err)
+	}
+	if got := stderr.String(); !strings.Contains(got, "\nCOMMAND EXITED NON-ZERO:\n") || strings.Contains(got, "FAILED:") {
+		t.Fatalf("expected command-result framing without FAILED header, got %q", got)
+	}
+}
+
 func TestRunMultiExecClassifiesTimeoutAndTransportFailures(t *testing.T) {
 	client := &fakePdshExecClient{
 		outputs: map[string]string{},
@@ -1027,3 +1063,30 @@ func TestResolveExecNoPrefix(t *testing.T) {
 		})
 	}
 }
+
+func TestRunMultiExecInfraFailureCarriesSentinel(t *testing.T) {
+	// Mixed remote-exit + timeout: any delivery failure keeps the FAILED
+	// framing and maps to the dedicated infra exit code so monitors can
+	// tell it apart from a command that merely exited non-zero.
+	client := &fakePdshExecClient{
+		errs: map[string]error{
+			"okdev-sess-worker-0": k8sexec.CodeExitError{Err: errors.New("command terminated with exit code 1"), Code: 1},
+			"okdev-sess-worker-1": context.DeadlineExceeded,
+		},
+	}
+	pods := []kube.PodSummary{
+		{Name: "okdev-sess-worker-0", Phase: "Running"},
+		{Name: "okdev-sess-worker-1", Phase: "Running"},
+	}
+	var stdout, stderr bytes.Buffer
+	err := runMultiExec(context.Background(), client, "default", pods, "dev", []string{"true"}, "", 0, false, make(chan struct{}, pdshDefaultFanout), &stdout, &stderr)
+	if err == nil || !errors.Is(err, ErrExecInfraFailure) {
+		t.Fatalf("expected infra sentinel, got %v", err)
+	}
+	if code, ok := ClassifiedExitCode(err); !ok || code != ExitExecInfraFailure {
+		t.Fatalf("expected exit code %d, got %d ok=%v", ExitExecInfraFailure, code, ok)
+	}
+	if !strings.Contains(stderr.String(), "FAILED:") {
+		t.Fatalf("expected FAILED framing for infra failures, got %q", stderr.String())
+	}
+}

internal/cli/sync.go

@@ -185,6 +185,7 @@ func newSyncCmd(opts *Options) *cobra.Command {
 	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Preview sync actions without transferring files")
 
 	cmd.AddCommand(newSyncResetRemoteCmd(opts))
+	cmd.AddCommand(newSyncWaitCmd(opts))
 	return cmd
 }