fix(ssh): use port 2222 for inter-pod SSH connections

The generated inter-pod SSH config was missing the Port directive, causing ssh to connect on port 22 while okdev-sshd listens on 2222. Also install openssh-client in the E2E test container and wait for it before attempting inter-pod SSH.

Author: acmore

internal/cli/ssh.go

@@ -460,6 +460,7 @@ func buildInterPodSSHConfig(user string, endpoints []interPodSSHEndpoint) string
 	for _, endpoint := range endpoints {
 		fmt.Fprintf(&b, "Host %s\n", endpoint.PodName)
 		fmt.Fprintf(&b, "  HostName %s\n", endpoint.PodIP)
+		fmt.Fprintf(&b, "  Port 2222\n")
 		fmt.Fprintf(&b, "  User %s\n", user)
 		fmt.Fprintln(&b, "  IdentityFile ~/.ssh/okdev_interpod_ed25519")
 		fmt.Fprintln(&b, "  IdentitiesOnly yes")

internal/cli/ssh_interpod_test.go

@@ -36,6 +36,7 @@ func TestBuildInterPodSSHConfigIncludesSessionPods(t *testing.T) {
 	for _, want := range []string{
 		"Host trainer-master-0",
 		"HostName 10.0.0.11",
+		"Port 2222",
 		"Host trainer-worker-0",
 		"HostName 10.0.0.12",
 		"IdentityFile ~/.ssh/okdev_interpod_ed25519",

scripts/e2e_kind_pytorchjob_interpod_ssh.sh

@@ -64,7 +64,7 @@ cd "$WORKDIR"
 MANIFEST_PATH="$WORKDIR/.okdev/pytorchjob.yaml"
 replace_all_in_file "$MANIFEST_PATH" 'name: dev' 'name: pytorch'
 replace_all_in_file "$MANIFEST_PATH" 'image: # TODO: replace with your image' 'image: ubuntu:22.04'
-replace_all_in_file "$MANIFEST_PATH" 'command: ["sleep", "infinity"]' 'command: ["sh", "-lc", "trap : TERM INT; while true; do sleep 3600; done"]'
+replace_all_in_file "$MANIFEST_PATH" 'command: ["sleep", "infinity"]' 'command: ["sh", "-lc", "apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1; trap : TERM INT; while true; do sleep 3600; done"]'
 replace_all_in_file "$CFG_PATH" 'container: dev' 'container: pytorch'
 
 python3 - <<'PY' "$CFG_PATH"
@@ -116,6 +116,18 @@ if [[ "$WORKER_CONTAINERS" != *"okdev-sidecar"* ]]; then
 fi
 echo "worker sidecar override verified"
 
+echo "Waiting for ssh client to be available in master pod"
+for i in $(seq 1 30); do
+  if "$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" ssh --cmd 'which ssh' >/dev/null 2>&1; then
+    break
+  fi
+  if [[ "$i" -eq 30 ]]; then
+    echo "ERROR: ssh client not available in master pod after 60s" >&2
+    exit 1
+  fi
+  sleep 2
+done
+
 echo "Verifying inter-pod SSH from master to worker"
 "$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" ssh --setup-key --cmd 'echo pytorchjob-ssh-ok' >/dev/null
 INTERPOD_SSH_OUTPUT=$("$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" ssh --cmd "ssh -o BatchMode=yes $FIRST_WORKER_POD 'echo interpod-ssh-ok'")