perf(onboard): build sandbox image with BuildKit + no-silent-progress heartbeats (#6002)

.github/workflows/e2e.yaml

@@ -2781,6 +2781,9 @@ jobs:
           npx vitest run --project e2e-live \
             test/e2e/live/full-e2e.test.ts \
             --silent=false --reporter=default
+          npx vitest run --project e2e-live \
+            test/e2e/live/onboard-progress-budget.test.ts \
+            --silent=false --reporter=default
 
       - name: Upload full-e2e artifacts
         if: always()

src/lib/build-context.test.ts

@@ -183,6 +183,40 @@ describe("printSandboxCreateRecoveryHints", () => {
     expect(out).toContain("<YOUR_RUNTIME_ENV>");
   });
 
+  it("shows the pushed image ref (not a Dockerfile path) when the BuildKit prebuild rewrote --from (#6002)", () => {
+    // After the BuildKit prebuild, createArgs carries `--from <local-image-ref>`
+    // — the Dockerfile path was rewritten away before create. On an upload-404
+    // the recovery command must still swap --from to the pushed registry ref,
+    // leaving no Dockerfile path and no stale prebuilt local ref as --from.
+    printSandboxCreateRecoveryHints(
+      [
+        "  Built image openshell/sandbox-from-nemoclaw:abcd1234",
+        "failed to upload image tar into container",
+      ].join("\n"),
+      {
+        platform: "linux",
+        arch: "x64",
+        createArgs: [
+          "--from",
+          "nemoclaw-sandbox-local:my-assistant-1234567890",
+          "--name",
+          "my-assistant",
+          "--policy",
+          "/tmp/nemoclaw-policy-xyz.yaml",
+        ],
+      },
+    );
+
+    const out = stderr();
+    // --from is the pushed registry ref derived from the build log's image tag,
+    expect(out).toContain("--from localhost:5000/openshell/sandbox-from-nemoclaw:abcd1234");
+    // never a Dockerfile path (the prebuild removed it) and never the stale
+    // prebuilt local ref left as the --from value.
+    expect(out).not.toContain("Dockerfile");
+    expect(out).not.toContain("--from nemoclaw-sandbox-local:my-assistant-1234567890");
+    expect(out).toContain("--name my-assistant");
+  });
+
   it("falls back to placeholder push commands when no built image tag is in the output", () => {
     printSandboxCreateRecoveryHints("failed to upload image tar into container", {
       platform: "linux",

src/lib/onboard.ts

@@ -746,6 +746,12 @@ const {
 
 // Gateway state functions — delegated to src/lib/state/gateway.ts
 const { isSandboxReady, parseSandboxStatus, getSandboxStateFromOutputs } = gatewayState;
+const waitForSandboxReady = sandboxReadinessTracing.createSandboxReadyWaiter({
+  runCaptureOpenshell,
+  isSandboxReady,
+  isLinuxDockerDriverGatewayEnabled,
+  sleep: sleepSeconds,
+});
 const { hasStaleGateway, isSelectedGateway, isGatewayHealthy, getGatewayReuseState } =
   gatewayBinding.createGatewayNameBoundClassifiers(gatewayState, () => GATEWAY_NAME);
 
@@ -1553,39 +1559,6 @@ async function ensureNamedCredential(
   return credentialPrompt.ensureNamedCredential(envName, label, helpUrl);
 }
 
-function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = 2): boolean {
-  for (let i = 0; i < attempts; i += 1) {
-    const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true });
-    if (isSandboxReady(list, sandboxName)) return true;
-
-    // Package-managed OpenShell gateways report readiness through
-    // `sandbox list`; legacy Kubernetes gateways may still expose pod state.
-    if (isLinuxDockerDriverGatewayEnabled()) {
-      if (i < attempts - 1) sleepSeconds(delaySeconds);
-      continue;
-    }
-    const podPhase = runCaptureOpenshell(
-      [
-        "doctor",
-        "exec",
-        "--",
-        "kubectl",
-        "-n",
-        "openshell",
-        "get",
-        "pod",
-        sandboxName,
-        "-o",
-        "jsonpath={.status.phase}",
-      ],
-      { ignoreError: true },
-    );
-    if (podPhase === "Running") return true;
-    sleepSeconds(delaySeconds);
-  }
-  return false;
-}
-
 // parsePolicyPresetEnv — see urlUtils import above
 // isSafeModelId — see validation import above
 
@@ -2940,8 +2913,8 @@ async function createSandbox(
     gatewayPort: GATEWAY_PORT,
   });
   const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(effectiveSandboxGpuConfig);
-  const { createCommand, effectiveDashboardPort, sandboxEnv, sandboxStartupCommand } =
-    sandboxCreateLaunch.prepareSandboxCreateLaunch({
+  const { createCommand, effectiveDashboardPort, prebuild, sandboxEnv, sandboxStartupCommand } =
+    await sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild({
       agent,
       chatUiUrl,
       createArgs,
@@ -2952,6 +2925,8 @@ async function createSandbox(
       hermesDashboardState,
       manageDashboard,
       openshellShellCommand,
+      // Transitional BuildKit handoff removal is tracked by #6258.
+      prebuild: { buildCtx, buildId, dockerDriverGateway: isLinuxDockerDriverGatewayEnabled() },
     });
   const dockerGpuCreatePatch = dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch({
     enabled: useDockerGpuPatch,
@@ -3013,7 +2988,7 @@ async function createSandbox(
         backupPath: restoreBackupPath,
       });
       console.error("  Try:  openshell sandbox list        # check gateway state");
-      printSandboxCreateRecoveryHints(createResult.output, { createArgs });
+      printSandboxCreateRecoveryHints(createResult.output, { createArgs: prebuild.createArgs });
       process.exit(createResult.status || 1);
     }
   }
@@ -3097,8 +3072,10 @@ async function createSandbox(
     hermesDashboardForwarding.ensureForState(finalHermesDashboardState, sandboxName, true);
   }
 
-  // Register only after ready; OpenShell tags in seconds, so parse the tag instead of using buildId.
-  const resolvedImageTag = resolveSandboxImageTagFromCreateOutput(createResult.output, buildId);
+  // Register only after confirmed ready — prevents phantom entries
+  // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672.
+  const resolvedImageTag =
+    prebuild.imageRef ?? resolveSandboxImageTagFromCreateOutput(createResult.output, buildId);
 
   const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig);
   const inferenceSelection = sandboxRegistration.selection;
@@ -4604,7 +4581,8 @@ async function preflightAuthoritativeRebuildTarget(
 }
 
 // ── Main ─────────────────────────────────────────────────────────
-async function onboard(opts: OnboardOptions = {}): Promise<void> {
+const onboard = onboardEntryOptions.withNonInteractiveEnvironment(runOnboard);
+async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
   const authoritativeGateway =
     authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts);
   const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT };

src/lib/onboard/entry-options.test.ts

@@ -3,7 +3,11 @@
 
 import { describe, expect, it, vi } from "vitest";
 
-import { type OnboardEntryOptionsDeps, resolveOnboardEntryOptions } from "./entry-options";
+import {
+  type OnboardEntryOptionsDeps,
+  resolveOnboardEntryOptions,
+  withNonInteractiveEnvironment,
+} from "./entry-options";
 
 class ExitError extends Error {
   constructor(readonly code: number) {
@@ -208,3 +212,46 @@ describe("resolveOnboardEntryOptions", () => {
     expect(deps.error).toHaveBeenCalledWith("  Use lowercase letters, numbers, and hyphens.");
   });
 });
+
+describe("withNonInteractiveEnvironment", () => {
+  it.each([
+    { label: "an unset value", env: {} as NodeJS.ProcessEnv, restored: undefined },
+    {
+      label: "an existing value",
+      env: { NEMOCLAW_NON_INTERACTIVE: "existing" } as NodeJS.ProcessEnv,
+      restored: "existing",
+    },
+  ])("sets the compatibility flag and restores $label", async ({ env, restored }) => {
+    const run = vi.fn(async () => {
+      expect(env.NEMOCLAW_NON_INTERACTIVE).toBe("1");
+    });
+
+    await withNonInteractiveEnvironment(run, env)({ nonInteractive: true });
+
+    expect(run).toHaveBeenCalledOnce();
+    expect(env.NEMOCLAW_NON_INTERACTIVE).toBe(restored);
+  });
+
+  it("restores the compatibility flag when onboarding rejects", async () => {
+    const env = {} as NodeJS.ProcessEnv;
+    const run = vi.fn(async () => {
+      throw new Error("onboarding failed");
+    });
+
+    await expect(withNonInteractiveEnvironment(run, env)({ nonInteractive: true })).rejects.toThrow(
+      "onboarding failed",
+    );
+    expect(env.NEMOCLAW_NON_INTERACTIVE).toBeUndefined();
+  });
+
+  it("passes options through without changing the environment when the flag is absent", async () => {
+    const env = { NEMOCLAW_NON_INTERACTIVE: "existing" } as NodeJS.ProcessEnv;
+    const options = { nonInteractive: false, marker: "unchanged" };
+    const run = vi.fn(async () => {});
+
+    await withNonInteractiveEnvironment(run, env)(options);
+
+    expect(run).toHaveBeenCalledWith(options);
+    expect(env.NEMOCLAW_NON_INTERACTIVE).toBe("existing");
+  });
+});

src/lib/onboard/entry-options.ts

@@ -43,6 +43,27 @@ export interface ResolvedOnboardEntryOptions {
   cannotPrompt: boolean;
 }
 
+type NonInteractiveEntryOptions = { nonInteractive?: boolean };
+
+/** Scope the CLI flag to helpers that still read the compatibility environment variable. */
+export function withNonInteractiveEnvironment<Options extends NonInteractiveEntryOptions>(
+  run: (options?: Options) => Promise<void>,
+  env: NodeJS.ProcessEnv = process.env,
+): (options?: Options) => Promise<void> {
+  return async (options) => {
+    if (options?.nonInteractive !== true) return run(options);
+
+    const previous = env.NEMOCLAW_NON_INTERACTIVE;
+    env.NEMOCLAW_NON_INTERACTIVE = "1";
+    try {
+      await run(options);
+    } finally {
+      if (previous === undefined) delete env.NEMOCLAW_NON_INTERACTIVE;
+      else env.NEMOCLAW_NON_INTERACTIVE = previous;
+    }
+  };
+}
+
 export function resolveOnboardEntryOptions(
   input: OnboardEntryOptionsInput,
   deps: OnboardEntryOptionsDeps,

src/lib/onboard/machine/live-flow-slice.test.ts

@@ -102,6 +102,7 @@ describe("runLiveOnboardFlowSlice", () => {
     const applyCompatibleResult = vi.fn(async (result: OnboardStateResult) =>
       liveRuntime.applyResult(result),
     );
+    const wrappedStates: string[] = [];
 
     const result = await runLiveOnboardFlowSlice({
       context: { value: 1 },
@@ -118,13 +119,20 @@ describe("runLiveOnboardFlowSlice", () => {
       ],
       runWhenState: ["preflight"],
       compatibilityWhenState: ["provider_selection"],
+      phaseProgress: {
+        wrap: (candidate) => {
+          wrappedStates.push(candidate.state);
+          return candidate;
+        },
+      },
       runSlice,
       applyCompatibleResult,
     });
 
     expect(result.context).toEqual({ value: 3 });
     expect(result.session.machine.state).toBe("inference");
     expect(runSlice).not.toHaveBeenCalled();
+    expect(wrappedStates).toEqual(["preflight", "gateway"]);
     expect(applyCompatibleResult.mock.calls.map(([result]) => result)).toEqual(results);
   });
 

src/lib/onboard/machine/live-flow-slice.ts

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { createPhaseProgressReporter, type PhaseProgressReporter } from "./phase-progress";
 import type { OnboardStateResult } from "./result";
 import type {
   OnboardMachineRunnerResult,
@@ -16,6 +17,7 @@ export interface LiveOnboardFlowSliceOptions<Context> {
   phases: readonly OnboardSequencePhase<Context>[];
   runWhenState: readonly OnboardMachineState[];
   compatibilityWhenState?: readonly OnboardMachineState[];
+  phaseProgress?: PhaseProgressReporter;
   runSlice(options: {
     context: Context;
     runtime: OnboardMachineRunnerRuntime;
@@ -78,6 +80,7 @@ export async function runLiveOnboardFlowSlice<Context>({
   phases,
   runWhenState,
   compatibilityWhenState = [],
+  phaseProgress = createPhaseProgressReporter(),
   runSlice,
   applyCompatibleResult,
 }: LiveOnboardFlowSliceOptions<Context>): Promise<OnboardMachineRunnerResult<Context>> {
@@ -98,7 +101,8 @@ export async function runLiveOnboardFlowSlice<Context>({
 
   assertUniquePhases(phases);
   let nextContext = context;
-  for (const phase of phases) {
+  for (const rawPhase of phases) {
+    const phase = phaseProgress.wrap(rawPhase);
     const phaseResult = await phase.run(nextContext);
     for (const result of asResultArray(phaseResult.result, phase.state)) {
       await applyCompatibleResult(result);