fix(truenas): wait for systemd boot completion in Ready

Add a deterministic readiness probe to TrueNAS Ready() that runs
`systemctl is-system-running --wait` inside the container, then `id`,
in a single SSH session. This blocks until systemd reports a stable
boot state instead of relying on client-side polling.

Background: Ready() previously declared success once SSH accepted the
connection and `ssh ... true` (TestAuth) returned 0. Freshly-booted
clones could still have unit activations in flight — agent reports
showed first exec returning exit 0 with empty stdout, with retries
working seconds later. Live testing measured ~90-103s from
create_sandbox to first successful exec, with the first exec attempt
timing out and the retry returning instantly. The systemd probe
absorbs that wait inside Ready().

`|| true` swallows the non-zero exit returned for "degraded" (non-
essential units that never start are normal); `id` then verifies the
user session is live and non-empty stdout returns from a real command.

Fixes #15

Author: deevus

sandbox/truenas/backend_test.go

@@ -672,7 +672,7 @@ func TestWriteFile(t *testing.T) {
 
 func TestReady(t *testing.T) {
 	t.Run("happy path: RUNNING with IP, auth ok", func(t *testing.T) {
-		mssh := &mockSSH{}
+		mssh := &mockSSH{outputFn: probeOK()}
 		tn, _ := NewForTest(&Client{
 			Virt: &tnapi.MockVirtService{
 				GetInstanceFunc: runningInstanceFunc("10.0.0.5"),
@@ -692,7 +692,7 @@ func TestReady(t *testing.T) {
 	})
 
 	t.Run("polls until RUNNING with IP", func(t *testing.T) {
-		mssh := &mockSSH{}
+		mssh := &mockSSH{outputFn: probeOK()}
 		var calls int
 		tn, _ := NewForTest(&Client{
 			Virt: &tnapi.MockVirtService{
@@ -728,6 +728,7 @@ func TestReady(t *testing.T) {
 			testAuthFn: func(ctx context.Context, cc ssh.ConnConfig) error {
 				return errors.New("permission denied")
 			},
+			outputFn: probeOK(),
 		}
 
 		dir := t.TempDir()
@@ -788,6 +789,77 @@ func TestReady(t *testing.T) {
 		}
 	})
 
+	t.Run("readiness probe runs systemctl + id in one ssh call", func(t *testing.T) {
+		mssh := &mockSSH{outputFn: probeOK()}
+		tn, _ := NewForTest(&Client{
+			Virt: &tnapi.MockVirtService{
+				GetInstanceFunc: runningInstanceFunc("10.0.0.5"),
+			},
+		}, mssh, testCfg())
+
+		if err := tn.Ready(context.Background(), "test", 5*time.Second); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if len(mssh.outputCalls) != 1 {
+			t.Fatalf("expected exactly one OutputQuiet call (single readiness probe); got %d", len(mssh.outputCalls))
+		}
+		probe := mssh.outputCalls[0]
+		if probe.User != "pixel" {
+			t.Errorf("probe user = %q, want pixel", probe.User)
+		}
+		if probe.Host != "px-test" {
+			t.Errorf("probe host = %q, want px-test", probe.Host)
+		}
+		if len(probe.Cmd) != 1 || !strings.Contains(probe.Cmd[0], "systemctl is-system-running --wait") {
+			t.Errorf("probe cmd = %v, want systemctl is-system-running --wait + id", probe.Cmd)
+		}
+		if !strings.Contains(probe.Cmd[0], "; id") {
+			t.Errorf("probe cmd = %v, missing trailing `id`", probe.Cmd)
+		}
+	})
+
+	t.Run("readiness probe empty output returns error", func(t *testing.T) {
+		mssh := &mockSSH{
+			outputFn: func(ctx context.Context, cc ssh.ConnConfig, cmd []string) ([]byte, error) {
+				return []byte(""), nil
+			},
+		}
+		tn, _ := NewForTest(&Client{
+			Virt: &tnapi.MockVirtService{
+				GetInstanceFunc: runningInstanceFunc("10.0.0.5"),
+			},
+		}, mssh, testCfg())
+
+		err := tn.Ready(context.Background(), "test", 5*time.Second)
+		if err == nil {
+			t.Fatal("expected error when readiness probe returns empty output")
+		}
+		if !strings.Contains(err.Error(), "empty output") {
+			t.Errorf("error %q should mention empty output", err.Error())
+		}
+	})
+
+	t.Run("readiness probe surfaces ssh error", func(t *testing.T) {
+		mssh := &mockSSH{
+			outputFn: func(ctx context.Context, cc ssh.ConnConfig, cmd []string) ([]byte, error) {
+				return nil, errors.New("connection reset")
+			},
+		}
+		tn, _ := NewForTest(&Client{
+			Virt: &tnapi.MockVirtService{
+				GetInstanceFunc: runningInstanceFunc("10.0.0.5"),
+			},
+		}, mssh, testCfg())
+
+		err := tn.Ready(context.Background(), "test", 5*time.Second)
+		if err == nil {
+			t.Fatal("expected ssh error to propagate")
+		}
+		if !strings.Contains(err.Error(), "connection reset") {
+			t.Errorf("error %q should wrap the ssh error", err.Error())
+		}
+	})
+
 	t.Run("instance never appears: deadline exceeded", func(t *testing.T) {
 		mssh := &mockSSH{}
 		tn, _ := NewForTest(&Client{

sandbox/truenas/exec.go

@@ -1,6 +1,7 @@
 package truenas
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -128,6 +129,32 @@ func (t *TrueNAS) Ready(ctx context.Context, name string, timeout time.Duration)
 			return fmt.Errorf("SSH key auth failed; writing key: %w", writeErr)
 		}
 	}
+
+	// Wait for systemd to reach a stable boot state. TestAuth confirms sshd
+	// accepts the connection, but freshly-booted clones can still have unit
+	// activations in flight — first exec has been seen returning exit 0 with
+	// empty stdout, recovering on retry. `systemctl is-system-running --wait`
+	// blocks inside the container until boot stabilises ("running" or
+	// "degraded"), giving us a deterministic readiness signal instead of
+	// client-side polling. `|| true` swallows the non-zero exit returned for
+	// "degraded" (non-essential units that never start are normal); `id` runs
+	// in the same SSH session afterward so we can verify the user's session
+	// is live and non-empty stdout returns from at least one real command.
+	remaining = time.Until(deadline)
+	if remaining <= 0 {
+		return fmt.Errorf("no time left for readiness probe on %s", name)
+	}
+	probeCtx, probeCancel := context.WithTimeout(ctx, remaining)
+	defer probeCancel()
+	out, err := t.ssh.OutputQuiet(probeCtx, cc, []string{
+		"systemctl is-system-running --wait >/dev/null 2>&1 || true; id",
+	})
+	if err != nil {
+		return fmt.Errorf("readiness probe on %s: %w", name, err)
+	}
+	if len(bytes.TrimSpace(out)) == 0 {
+		return fmt.Errorf("readiness probe on %s returned empty output", name)
+	}
 	return nil
 }
 

sandbox/truenas/network_test.go

@@ -58,6 +58,14 @@ func (m *mockSSH) OutputQuiet(ctx context.Context, cc ssh.ConnConfig, cmd []stri
 	return nil, nil
 }
 
+// probeOK returns an outputFn that satisfies Ready()'s smoke probe by
+// returning a non-empty `id`-style line for any call.
+func probeOK() func(ctx context.Context, cc ssh.ConnConfig, cmd []string) ([]byte, error) {
+	return func(ctx context.Context, cc ssh.ConnConfig, cmd []string) ([]byte, error) {
+		return []byte("uid=1000(pixel) gid=1000(pixel)\n"), nil
+	}
+}
+
 func (m *mockSSH) WaitReady(ctx context.Context, host string, timeout time.Duration, log io.Writer) error {
 	m.waitCalls = append(m.waitCalls, host)
 	if m.waitFn != nil {