Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/command-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ agents can react without launching a diagnostic chain on every blip:
- `okdev validate`
- `okdev up [--wait-timeout 10m] [--dry-run]`
- `okdev down [session] [--delete-pvc] [--dry-run] [--wait] [--wait-timeout 2m] [--output json]`
- `okdev restart [session] [--yes] [--wait-timeout 10m]`
- `okdev status [session] [--all] [--all-users] [--details]`
- `okdev list [--all-namespaces] [--all-users]`
- `okdev use <session>`
Expand Down Expand Up @@ -90,6 +91,7 @@ agents can react without launching a diagnostic chain on every blip:
- With `-- command...`, runs the command across all running pods by default, with output prefixed by short pod name on a TTY. When stdout is piped or redirected (scripts, CI, agents), the prefix is auto-suppressed so captured output is clean; pass `--no-prefix=false` to force it back on.
- `--all` explicitly targets all session pods. This matches the existing default fanout behavior when you pass a command with no selector.
- `--workers` targets only worker-role pods.
- `--pod` accepts the full pod name, the short name shown by `okdev status` (e.g. `master-0`, `worker-1`), or any unique `-<name>` suffix — no need to carry the full hash name across pod recreations. Unknown or ambiguous names error out (listing the available names) instead of silently matching nothing. `--exclude` accepts the same aliases (unknown exclusions are tolerated). The same aliases work for `okdev cp --pod`, `okdev jobs ... --pod`, and `okdev target set --pod`.
- `--group <pod-a,pod-b>` targets an explicit pod group. Repeat `--group` to define multiple groups, and use either full pod names or the short names shown in `okdev status`. A pod may appear in at most one group.
- Declared groups run one after another by default.
- `--sequential` runs selected pods one-by-one when no groups are declared (output and `--log-dir` layout are unchanged), or makes the default group-after-group ordering explicit when groups are declared.
Expand Down Expand Up @@ -261,6 +263,14 @@ agents can react without launching a diagnostic chain on every blip:
- `--output json`: emits a machine-readable summary of the planned or completed deletion and local cleanup steps.
- `--delete-pvc` remains accepted for compatibility but is ignored; `okdev` no longer manages PVC lifecycle automatically.

### `okdev restart [session] [--yes] [--wait-timeout 10m]`

- One-command recovery when a container died and the restart policy will not bring it back: deletes the session workload, waits for termination, resets local per-session sync state, and runs the full `up` flow against the current config.
- PVCs and other resources referenced by `spec.volumes` are untouched; auto-provisioned sync volumes are recreated.
- Lifecycle hooks (`postCreate`/`postSync`) re-run automatically on the fresh pods — configure them for setup that must survive recreation (installed tools, editable installs).
- Pod names change on recreation; use the short-name aliases (`master-0`, `worker-1`) with `--pod` so scripts survive restarts.
- Prompts for confirmation; `--yes` for scripts. `--wait-timeout` caps both the deletion wait and pod readiness.

### `okdev ssh [session] [--setup-key] [--user root] [--cmd "..."] [--no-tmux] [--forward-agent|--no-forward-agent]`

- Targets `okdev-sshd` in the resolved interactive container.
Expand Down
6 changes: 5 additions & 1 deletion internal/cli/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,11 @@ func runMultiPodCp(cmd *cobra.Command, cc *commandContext, localPath, remotePath
case allPods:
// keep all
case len(podNames) > 0:
filteredPods = filterPodsByName(filteredPods, podNames)
selected, err := resolvePodAliases(filteredPods, podNames)
if err != nil {
return err
}
filteredPods = selected
case role != "":
filteredPods = filterPodsByRole(filteredPods, role)
case len(labels) > 0:
Expand Down
80 changes: 66 additions & 14 deletions internal/cli/pdsh.go
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,11 @@ func buildExecPodGroups(sessionName string, sessionPods []kube.PodSummary, plan
filtered := sessionPods
switch {
case len(plan.podNames) > 0:
filtered = filterPodsByName(filtered, plan.podNames)
selected, err := resolvePodAliases(filtered, plan.podNames)
if err != nil {
return nil, err
}
filtered = selected
case plan.workers:
filtered = filterPodsByRole(filtered, "worker")
case plan.role != "":
Expand Down Expand Up @@ -1645,28 +1649,76 @@ func filterPodsByLabels(pods []kube.PodSummary, labels []string) []kube.PodSumma
return out
}

func filterPodsByName(pods []kube.PodSummary, names []string) []kube.PodSummary {
nameSet := make(map[string]bool, len(names))
for _, n := range names {
nameSet[n] = true
}
out := make([]kube.PodSummary, 0)
for _, p := range pods {
if nameSet[p.Name] {
out = append(out, p)
// resolvePodAliases maps requested pod names to session pods. Accepted
// forms, in order: the full pod name, the short name shown by `okdev
// status` (longest common prefix stripped — e.g. "master-0", "worker-1"),
// or a unique "-<name>" suffix. Unknown or ambiguous names error instead of
// being silently dropped.
func resolvePodAliases(sessionPods []kube.PodSummary, names []string) ([]kube.PodSummary, error) {
fullNames := podSummaryNames(sessionPods)
shorts := shortPodNames(fullNames)
byAlias := make(map[string]kube.PodSummary, len(sessionPods)*2)
for i, pod := range sessionPods {
byAlias[pod.Name] = pod
byAlias[shorts[i]] = pod
}
out := make([]kube.PodSummary, 0, len(names))
seen := make(map[string]bool, len(names))
for _, name := range names {
name = strings.TrimSpace(name)
if name == "" {
continue
}
pod, ok := byAlias[name]
if !ok {
var suffixMatches []kube.PodSummary
for _, p := range sessionPods {
if strings.HasSuffix(p.Name, "-"+name) {
suffixMatches = append(suffixMatches, p)
}
}
switch len(suffixMatches) {
case 1:
pod = suffixMatches[0]
case 0:
return nil, fmt.Errorf("no session pod matches %q; available: %s", name, strings.Join(shorts, ", "))
default:
return nil, fmt.Errorf("pod name %q is ambiguous: %s", name, strings.Join(podSummaryNames(suffixMatches), ", "))
}
}
if seen[pod.Name] {
continue
}
seen[pod.Name] = true
out = append(out, pod)
}
return out
return out, nil
}

// excludePods drops the named pods, accepting the same aliases as
// resolvePodAliases (full name, status short name, "-<name>" suffix).
// Unknown exclusions are tolerated: excluding a pod that no longer exists
// should not fail a fanout.
func excludePods(pods []kube.PodSummary, exclude []string) []kube.PodSummary {
shorts := shortPodNames(podSummaryNames(pods))
excludeSet := make(map[string]bool, len(exclude))
for _, n := range exclude {
excludeSet[n] = true
if n = strings.TrimSpace(n); n != "" {
excludeSet[n] = true
}
}
out := make([]kube.PodSummary, 0, len(pods))
for _, p := range pods {
if !excludeSet[p.Name] {
for i, p := range pods {
excluded := excludeSet[p.Name] || excludeSet[shorts[i]]
if !excluded {
for n := range excludeSet {
if strings.HasSuffix(p.Name, "-"+n) {
excluded = true
break
}
}
}
if !excluded {
out = append(out, p)
}
}
Expand Down
53 changes: 41 additions & 12 deletions internal/cli/pdsh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,25 +46,54 @@ func TestFilterPodsByLabels(t *testing.T) {
}
}

func TestFilterPodsByName(t *testing.T) {
func TestResolvePodAliases(t *testing.T) {
pods := []kube.PodSummary{
{Name: "worker-0"},
{Name: "worker-1"},
{Name: "master-0"},
{Name: "trainjob-abc-master-0"},
{Name: "trainjob-abc-worker-0"},
{Name: "trainjob-abc-worker-1"},
}
got := filterPodsByName(pods, []string{"worker-0", "master-0"})
if len(got) != 2 {
t.Fatalf("expected 2 pods, got %d", len(got))

// Full names, status short names, and unique suffixes all resolve.
got, err := resolvePodAliases(pods, []string{"trainjob-abc-master-0", "worker-1"})
if err != nil || len(got) != 2 || got[0].Name != "trainjob-abc-master-0" || got[1].Name != "trainjob-abc-worker-1" {
t.Fatalf("unexpected resolution: %v err=%v", got, err)
}
got, err = resolvePodAliases(pods, []string{"master-0"})
if err != nil || len(got) != 1 || got[0].Name != "trainjob-abc-master-0" {
t.Fatalf("short name must resolve, got %v err=%v", got, err)
}

// Duplicates collapse.
got, err = resolvePodAliases(pods, []string{"worker-0", "trainjob-abc-worker-0"})
if err != nil || len(got) != 1 {
t.Fatalf("expected dedup, got %v err=%v", got, err)
}

// Unknown names error with available aliases instead of silently
// dropping (the old filter returned a smaller set).
if _, err := resolvePodAliases(pods, []string{"worker-9"}); err == nil || !strings.Contains(err.Error(), "master-0") {
t.Fatalf("expected unknown-name error listing aliases, got %v", err)
}

// Ambiguous suffixes error and list candidates.
if _, err := resolvePodAliases(pods, []string{"0"}); err == nil || !strings.Contains(err.Error(), "ambiguous") {
t.Fatalf("expected ambiguity error, got %v", err)
}
}

func TestFilterPodsByNameMissing(t *testing.T) {
func TestExcludePodsAliases(t *testing.T) {
pods := []kube.PodSummary{
{Name: "worker-0"},
{Name: "trainjob-abc-master-0"},
{Name: "trainjob-abc-worker-0"},
}
got := excludePods(pods, []string{"worker-0"})
if len(got) != 1 || got[0].Name != "trainjob-abc-master-0" {
t.Fatalf("short-name exclusion failed: %v", got)
}
got := filterPodsByName(pods, []string{"worker-0", "no-such-pod"})
if len(got) != 1 {
t.Fatalf("expected 1 pod, got %d", len(got))
// Unknown exclusions are tolerated (pod may no longer exist).
got = excludePods(pods, []string{"gone-pod"})
if len(got) != 2 {
t.Fatalf("unknown exclusion must be a no-op, got %v", got)
}
}

Expand Down
117 changes: 117 additions & 0 deletions internal/cli/restart.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package cli

import (
"bufio"
"fmt"
"io"
"os"
"strings"
"time"

"github.com/spf13/cobra"
)

func newRestartCmd(opts *Options) *cobra.Command {
var yes bool
var waitTimeout time.Duration

cmd := &cobra.Command{
Use: "restart [session]",
Short: "Recreate the session workload with the same config",
Long: `Delete the session workload, wait for it to terminate, and bring it back
up from the current config in one command — the recovery path when a
container died and the restart policy will not bring it back.

PVCs and other resources referenced by spec.volumes are untouched.
Lifecycle hooks (postCreate/postSync) re-run automatically on the fresh
pods, and sync re-bootstraps against the new workload.`,
Args: cobra.MaximumNArgs(1),
ValidArgsFunction: sessionCompletionFunc(opts),
Example: ` # Restart the current session (prompts for confirmation)
okdev restart

# Restart without confirmation
okdev restart --yes

# Restart a specific session
okdev restart my-feature -y`,
RunE: func(cmd *cobra.Command, args []string) error {
applySessionArg(opts, args)
ui := newUpUI(cmd.OutOrStdout(), cmd.ErrOrStderr())
ui.section("Validate")
cc, err := resolveCommandContext(opts, resolveSessionName)
if err != nil {
return err
}
ui.stepDone("config", cc.cfgPath)
ui.stepDone("session", cc.sessionName)
ui.stepDone("namespace", cc.namespace)
runtime, err := sessionRuntimeForExisting(cmd.Context(), cc.cfg, cc.cfgPath, cc.namespace, cc.sessionName, cc.kube)
if err != nil {
return err
}
if err := ensureSessionOwnership(cc.opts, cc.kube, cc.namespace, cc.sessionName); err != nil {
return err
}
ui.stepDone("ownership", "ok")

delCtx, cancel := downCommandContext(true, waitTimeout)
defer cancel()
exists, err := shouldReuseExistingWorkload(delCtx, cc.kube, cc.namespace, runtime)
if err != nil {
return err
}

if exists {
if !yes {
ok, err := confirmRestart(os.Stdin, cmd.ErrOrStderr(), cc.sessionName, cc.namespace, runtime.Kind(), runtime.WorkloadName())
if err != nil {
return err
}
if !ok {
fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")
return nil
}
}
ui.section("Delete")
if err := runtime.Delete(delCtx, cc.kube, cc.namespace, true); err != nil {
return fmt.Errorf("delete workload: %w", err)
}
ui.stepRun("workload", "waiting for deletion")
if _, err := waitForDownDeletion(delCtx, cc.kube, cc.namespace, cc.sessionName, runtime, waitTimeout); err != nil {
return err
}
ui.stepDone("workload", fmt.Sprintf("%s/%s deleted", runtime.Kind(), runtime.WorkloadName()))
} else {
ui.stepDone("workload", "already absent")
}

// Reset local per-session state (sync daemons, pins) so the up
// below bootstraps cleanly against the fresh workload.
payload := downOutput{Cleanup: map[string]any{}}
if err := downCleanupLocal(ui, &payload, cc.sessionName); err != nil {
return err
}

return runUp(cmd, opts, upOptions{waitTimeout: waitTimeout})
},
}

cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt")
cmd.Flags().DurationVar(&waitTimeout, "wait-timeout", upDefaultWaitTimeout, "Timeout for workload deletion and pod readiness")
return cmd
}

func confirmRestart(in io.Reader, out io.Writer, sessionName, namespace, kind, workloadName string) (bool, error) {
if !isTerminalReader(in) {
return false, fmt.Errorf("refusing to restart without --yes in non-interactive mode")
}
fmt.Fprintf(out, "Restart session %q in namespace %q? This deletes and recreates %s/%s. [y/N]: ", sessionName, namespace, kind, workloadName)
reader := bufio.NewReader(in)
line, err := reader.ReadString('\n')
if err != nil {
return false, fmt.Errorf("read confirmation: %w", err)
}
answer := strings.TrimSpace(strings.ToLower(line))
return answer == "y" || answer == "yes", nil
}
33 changes: 33 additions & 0 deletions internal/cli/restart_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package cli

import (
"bytes"
"strings"
"testing"
)

func TestNewRestartCmdFlags(t *testing.T) {
cmd := newRestartCmd(&Options{})
if cmd.Use != "restart [session]" {
t.Fatalf("unexpected use: %q", cmd.Use)
}
yes := cmd.Flags().Lookup("yes")
if yes == nil || yes.Shorthand != "y" {
t.Fatalf("expected --yes/-y flag, got %+v", yes)
}
wt := cmd.Flags().Lookup("wait-timeout")
if wt == nil || wt.DefValue != upDefaultWaitTimeout.String() {
t.Fatalf("expected --wait-timeout default %s, got %+v", upDefaultWaitTimeout, wt)
}
}

func TestConfirmRestartRefusesNonInteractive(t *testing.T) {
var out bytes.Buffer
ok, err := confirmRestart(strings.NewReader("y\n"), &out, "sess", "default", "Pod", "okdev-sess")
if err == nil || ok {
t.Fatalf("expected non-interactive refusal, got ok=%v err=%v", ok, err)
}
if !strings.Contains(err.Error(), "--yes") {
t.Fatalf("error should point at --yes, got %v", err)
}
}
1 change: 1 addition & 0 deletions internal/cli/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func newRootCmdWithOptions() (*cobra.Command, *Options) {
cmd.AddCommand(newValidateCmd(opts))
cmd.AddCommand(newUpCmd(opts))
cmd.AddCommand(newDownCmd(opts))
cmd.AddCommand(newRestartCmd(opts))
cmd.AddCommand(newStatusCmd(opts))
cmd.AddCommand(newListCmd(opts))
cmd.AddCommand(newUseCmd(opts))
Expand Down
Loading
Loading