diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5bd6eaa4a0..723ac1e68b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2781,6 +2781,9 @@ jobs: npx vitest run --project e2e-live \ test/e2e/live/full-e2e.test.ts \ --silent=false --reporter=default + npx vitest run --project e2e-live \ + test/e2e/live/onboard-progress-budget.test.ts \ + --silent=false --reporter=default - name: Upload full-e2e artifacts if: always() diff --git a/src/lib/build-context.test.ts b/src/lib/build-context.test.ts index 897c13e730..f290f019d7 100644 --- a/src/lib/build-context.test.ts +++ b/src/lib/build-context.test.ts @@ -183,6 +183,40 @@ describe("printSandboxCreateRecoveryHints", () => { expect(out).toContain(""); }); + it("shows the pushed image ref (not a Dockerfile path) when the BuildKit prebuild rewrote --from (#6002)", () => { + // After the BuildKit prebuild, createArgs carries `--from ` + // — the Dockerfile path was rewritten away before create. On an upload-404 + // the recovery command must still swap --from to the pushed registry ref, + // leaving no Dockerfile path and no stale prebuilt local ref as --from. + printSandboxCreateRecoveryHints( + [ + " Built image openshell/sandbox-from-nemoclaw:abcd1234", + "failed to upload image tar into container", + ].join("\n"), + { + platform: "linux", + arch: "x64", + createArgs: [ + "--from", + "nemoclaw-sandbox-local:my-assistant-1234567890", + "--name", + "my-assistant", + "--policy", + "/tmp/nemoclaw-policy-xyz.yaml", + ], + }, + ); + + const out = stderr(); + // --from is the pushed registry ref derived from the build log's image tag, + expect(out).toContain("--from localhost:5000/openshell/sandbox-from-nemoclaw:abcd1234"); + // never a Dockerfile path (the prebuild removed it) and never the stale + // prebuilt local ref left as the --from value. + expect(out).not.toContain("Dockerfile"); + expect(out).not.toContain("--from nemoclaw-sandbox-local:my-assistant-1234567890"); + expect(out).toContain("--name my-assistant"); + }); + it("falls back to placeholder push commands when no built image tag is in the output", () => { printSandboxCreateRecoveryHints("failed to upload image tar into container", { platform: "linux", diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index afb39d2be8..cb6a4f9872 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -746,6 +746,12 @@ const { // Gateway state functions — delegated to src/lib/state/gateway.ts const { isSandboxReady, parseSandboxStatus, getSandboxStateFromOutputs } = gatewayState; +const waitForSandboxReady = sandboxReadinessTracing.createSandboxReadyWaiter({ + runCaptureOpenshell, + isSandboxReady, + isLinuxDockerDriverGatewayEnabled, + sleep: sleepSeconds, +}); const { hasStaleGateway, isSelectedGateway, isGatewayHealthy, getGatewayReuseState } = gatewayBinding.createGatewayNameBoundClassifiers(gatewayState, () => GATEWAY_NAME); @@ -1553,39 +1559,6 @@ async function ensureNamedCredential( return credentialPrompt.ensureNamedCredential(envName, label, helpUrl); } -function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = 2): boolean { - for (let i = 0; i < attempts; i += 1) { - const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - if (isSandboxReady(list, sandboxName)) return true; - - // Package-managed OpenShell gateways report readiness through - // `sandbox list`; legacy Kubernetes gateways may still expose pod state. - if (isLinuxDockerDriverGatewayEnabled()) { - if (i < attempts - 1) sleepSeconds(delaySeconds); - continue; - } - const podPhase = runCaptureOpenshell( - [ - "doctor", - "exec", - "--", - "kubectl", - "-n", - "openshell", - "get", - "pod", - sandboxName, - "-o", - "jsonpath={.status.phase}", - ], - { ignoreError: true }, - ); - if (podPhase === "Running") return true; - sleepSeconds(delaySeconds); - } - return false; -} - // parsePolicyPresetEnv — see urlUtils import above // isSafeModelId — see validation import above @@ -2940,8 +2913,8 @@ async function createSandbox( gatewayPort: GATEWAY_PORT, }); const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(effectiveSandboxGpuConfig); - const { createCommand, effectiveDashboardPort, sandboxEnv, sandboxStartupCommand } = - sandboxCreateLaunch.prepareSandboxCreateLaunch({ + const { createCommand, effectiveDashboardPort, prebuild, sandboxEnv, sandboxStartupCommand } = + await sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild({ agent, chatUiUrl, createArgs, @@ -2952,6 +2925,8 @@ async function createSandbox( hermesDashboardState, manageDashboard, openshellShellCommand, + // Transitional BuildKit handoff removal is tracked by #6258. + prebuild: { buildCtx, buildId, dockerDriverGateway: isLinuxDockerDriverGatewayEnabled() }, }); const dockerGpuCreatePatch = dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch({ enabled: useDockerGpuPatch, @@ -3013,7 +2988,7 @@ async function createSandbox( backupPath: restoreBackupPath, }); console.error(" Try: openshell sandbox list # check gateway state"); - printSandboxCreateRecoveryHints(createResult.output, { createArgs }); + printSandboxCreateRecoveryHints(createResult.output, { createArgs: prebuild.createArgs }); process.exit(createResult.status || 1); } } @@ -3097,8 +3072,10 @@ async function createSandbox( hermesDashboardForwarding.ensureForState(finalHermesDashboardState, sandboxName, true); } - // Register only after ready; OpenShell tags in seconds, so parse the tag instead of using buildId. - const resolvedImageTag = resolveSandboxImageTagFromCreateOutput(createResult.output, buildId); + // Register only after confirmed ready — prevents phantom entries + // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. + const resolvedImageTag = + prebuild.imageRef ?? resolveSandboxImageTagFromCreateOutput(createResult.output, buildId); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); const inferenceSelection = sandboxRegistration.selection; @@ -4604,7 +4581,8 @@ async function preflightAuthoritativeRebuildTarget( } // ── Main ───────────────────────────────────────────────────────── -async function onboard(opts: OnboardOptions = {}): Promise { +const onboard = onboardEntryOptions.withNonInteractiveEnvironment(runOnboard); +async function runOnboard(opts: OnboardOptions = {}): Promise { const authoritativeGateway = authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT }; diff --git a/src/lib/onboard/entry-options.test.ts b/src/lib/onboard/entry-options.test.ts index ab91aa0aaa..83a9fc2a45 100644 --- a/src/lib/onboard/entry-options.test.ts +++ b/src/lib/onboard/entry-options.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it, vi } from "vitest"; -import { type OnboardEntryOptionsDeps, resolveOnboardEntryOptions } from "./entry-options"; +import { + type OnboardEntryOptionsDeps, + resolveOnboardEntryOptions, + withNonInteractiveEnvironment, +} from "./entry-options"; class ExitError extends Error { constructor(readonly code: number) { @@ -208,3 +212,46 @@ describe("resolveOnboardEntryOptions", () => { expect(deps.error).toHaveBeenCalledWith(" Use lowercase letters, numbers, and hyphens."); }); }); + +describe("withNonInteractiveEnvironment", () => { + it.each([ + { label: "an unset value", env: {} as NodeJS.ProcessEnv, restored: undefined }, + { + label: "an existing value", + env: { NEMOCLAW_NON_INTERACTIVE: "existing" } as NodeJS.ProcessEnv, + restored: "existing", + }, + ])("sets the compatibility flag and restores $label", async ({ env, restored }) => { + const run = vi.fn(async () => { + expect(env.NEMOCLAW_NON_INTERACTIVE).toBe("1"); + }); + + await withNonInteractiveEnvironment(run, env)({ nonInteractive: true }); + + expect(run).toHaveBeenCalledOnce(); + expect(env.NEMOCLAW_NON_INTERACTIVE).toBe(restored); + }); + + it("restores the compatibility flag when onboarding rejects", async () => { + const env = {} as NodeJS.ProcessEnv; + const run = vi.fn(async () => { + throw new Error("onboarding failed"); + }); + + await expect(withNonInteractiveEnvironment(run, env)({ nonInteractive: true })).rejects.toThrow( + "onboarding failed", + ); + expect(env.NEMOCLAW_NON_INTERACTIVE).toBeUndefined(); + }); + + it("passes options through without changing the environment when the flag is absent", async () => { + const env = { NEMOCLAW_NON_INTERACTIVE: "existing" } as NodeJS.ProcessEnv; + const options = { nonInteractive: false, marker: "unchanged" }; + const run = vi.fn(async () => {}); + + await withNonInteractiveEnvironment(run, env)(options); + + expect(run).toHaveBeenCalledWith(options); + expect(env.NEMOCLAW_NON_INTERACTIVE).toBe("existing"); + }); +}); diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index 313322ca0e..97c80328fd 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -43,6 +43,27 @@ export interface ResolvedOnboardEntryOptions { cannotPrompt: boolean; } +type NonInteractiveEntryOptions = { nonInteractive?: boolean }; + +/** Scope the CLI flag to helpers that still read the compatibility environment variable. */ +export function withNonInteractiveEnvironment( + run: (options?: Options) => Promise, + env: NodeJS.ProcessEnv = process.env, +): (options?: Options) => Promise { + return async (options) => { + if (options?.nonInteractive !== true) return run(options); + + const previous = env.NEMOCLAW_NON_INTERACTIVE; + env.NEMOCLAW_NON_INTERACTIVE = "1"; + try { + await run(options); + } finally { + if (previous === undefined) delete env.NEMOCLAW_NON_INTERACTIVE; + else env.NEMOCLAW_NON_INTERACTIVE = previous; + } + }; +} + export function resolveOnboardEntryOptions( input: OnboardEntryOptionsInput, deps: OnboardEntryOptionsDeps, diff --git a/src/lib/onboard/machine/live-flow-slice.test.ts b/src/lib/onboard/machine/live-flow-slice.test.ts index caae774217..0716c90f93 100644 --- a/src/lib/onboard/machine/live-flow-slice.test.ts +++ b/src/lib/onboard/machine/live-flow-slice.test.ts @@ -102,6 +102,7 @@ describe("runLiveOnboardFlowSlice", () => { const applyCompatibleResult = vi.fn(async (result: OnboardStateResult) => liveRuntime.applyResult(result), ); + const wrappedStates: string[] = []; const result = await runLiveOnboardFlowSlice({ context: { value: 1 }, @@ -118,6 +119,12 @@ describe("runLiveOnboardFlowSlice", () => { ], runWhenState: ["preflight"], compatibilityWhenState: ["provider_selection"], + phaseProgress: { + wrap: (candidate) => { + wrappedStates.push(candidate.state); + return candidate; + }, + }, runSlice, applyCompatibleResult, }); @@ -125,6 +132,7 @@ describe("runLiveOnboardFlowSlice", () => { expect(result.context).toEqual({ value: 3 }); expect(result.session.machine.state).toBe("inference"); expect(runSlice).not.toHaveBeenCalled(); + expect(wrappedStates).toEqual(["preflight", "gateway"]); expect(applyCompatibleResult.mock.calls.map(([result]) => result)).toEqual(results); }); diff --git a/src/lib/onboard/machine/live-flow-slice.ts b/src/lib/onboard/machine/live-flow-slice.ts index 883f507de7..aa497daaec 100644 --- a/src/lib/onboard/machine/live-flow-slice.ts +++ b/src/lib/onboard/machine/live-flow-slice.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createPhaseProgressReporter, type PhaseProgressReporter } from "./phase-progress"; import type { OnboardStateResult } from "./result"; import type { OnboardMachineRunnerResult, @@ -16,6 +17,7 @@ export interface LiveOnboardFlowSliceOptions { phases: readonly OnboardSequencePhase[]; runWhenState: readonly OnboardMachineState[]; compatibilityWhenState?: readonly OnboardMachineState[]; + phaseProgress?: PhaseProgressReporter; runSlice(options: { context: Context; runtime: OnboardMachineRunnerRuntime; @@ -78,6 +80,7 @@ export async function runLiveOnboardFlowSlice({ phases, runWhenState, compatibilityWhenState = [], + phaseProgress = createPhaseProgressReporter(), runSlice, applyCompatibleResult, }: LiveOnboardFlowSliceOptions): Promise> { @@ -98,7 +101,8 @@ export async function runLiveOnboardFlowSlice({ assertUniquePhases(phases); let nextContext = context; - for (const phase of phases) { + for (const rawPhase of phases) { + const phase = phaseProgress.wrap(rawPhase); const phaseResult = await phase.run(nextContext); for (const result of asResultArray(phaseResult.result, phase.state)) { await applyCompatibleResult(result); diff --git a/src/lib/onboard/machine/phase-progress.test.ts b/src/lib/onboard/machine/phase-progress.test.ts new file mode 100644 index 0000000000..2ff464596d --- /dev/null +++ b/src/lib/onboard/machine/phase-progress.test.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + createPhaseProgressReporter, + ONBOARD_PHASE_LABELS, + type PhaseProgressOptions, +} from "./phase-progress"; +import { advanceTo } from "./result"; +import type { OnboardSequencePhase, OnboardSequencePhaseResult } from "./sequence-runner"; + +function phase( + state: OnboardSequencePhase["state"], + run: (context: string) => Promise>, +): OnboardSequencePhase { + return { state, run }; +} + +function createHarness(overrides: Partial = {}) { + const state = { + clockMs: 0, + timerCallback: null as (() => void) | null, + timerIntervalMs: null as number | null, + cleared: false, + lines: [] as string[], + }; + const reporter = createPhaseProgressReporter({ + enabled: true, + heartbeatIntervalMs: 30_000, + now: () => state.clockMs, + setTimer: (callback, intervalMs) => { + state.timerCallback = callback; + state.timerIntervalMs = intervalMs; + return { unref() {} }; + }, + clearTimer: () => { + state.cleared = true; + }, + logLine: (line) => state.lines.push(line), + ...overrides, + }); + return { reporter, state }; +} + +describe("phase progress", () => { + it("returns phases unchanged when disabled or not wait-heavy", () => { + const original = phase("preflight", async (context) => ({ + context, + result: advanceTo("gateway"), + })); + expect(createPhaseProgressReporter({ enabled: false }).wrap(original)).toBe(original); + expect(createPhaseProgressReporter({ enabled: true }).wrap(original)).toBe(original); + }); + + it("emits a periodic heartbeat and always clears its timer", async () => { + const { reporter, state } = createHarness(); + const wrapped = reporter.wrap( + phase("gateway", async (context) => { + state.clockMs = 30_000; + state.timerCallback?.(); + return { context, result: advanceTo("provider_selection") }; + }), + ); + + await wrapped.run("ctx"); + + expect(state.timerIntervalMs).toBe(30_000); + expect(state.lines).toEqual([" ⏳ Still working on Gateway startup… (30s elapsed)"]); + expect(state.cleared).toBe(true); + }); + + it.each([ + "gateway", + "inference", + "sandbox", + "agent_setup", + "openclaw", + "finalizing", + "post_verify", + ] as const)("keeps wait-heavy phase %s on the heartbeat path", async (stateName) => { + const { reporter, state } = createHarness(); + await reporter + .wrap( + phase(stateName, async (context) => ({ + context, + result: advanceTo("post_verify"), + })), + ) + .run("ctx"); + expect(state.timerCallback).not.toBeNull(); + }); + + it.each([ + "provider_selection", + "policies", + ] as const)("protects the interactive %s prompt from heartbeat output", async (stateName) => { + const { reporter, state } = createHarness({ interactive: true }); + await reporter + .wrap( + phase(stateName, async (context) => ({ + context, + result: advanceTo("post_verify"), + })), + ) + .run("ctx"); + expect(state.timerCallback).toBeNull(); + }); + + it("heartbeats prompt-owning phases during non-interactive onboarding", async () => { + const { reporter, state } = createHarness({ interactive: false }); + await reporter + .wrap( + phase("provider_selection", async (context) => ({ + context, + result: advanceTo("inference"), + })), + ) + .run("ctx"); + expect(state.timerCallback).not.toBeNull(); + }); + + it("clears the timer and preserves the phase error", async () => { + const { reporter, state } = createHarness(); + const wrapped = reporter.wrap( + phase("gateway", async () => { + throw new Error("gateway exploded"); + }), + ); + await expect(wrapped.run("ctx")).rejects.toThrow("gateway exploded"); + expect(state.cleared).toBe(true); + }); + + it("keeps heartbeat logging best-effort", async () => { + const { reporter, state } = createHarness({ + logLine: () => { + throw new Error("closed output"); + }, + }); + await reporter + .wrap( + phase("gateway", async (context) => { + state.clockMs = 30_000; + expect(() => state.timerCallback?.()).not.toThrow(); + return { context, result: advanceTo("provider_selection") }; + }), + ) + .run("ctx"); + }); + + it("supports a shorter heartbeat interval for focused tests", async () => { + const valid = createHarness({ + heartbeatIntervalMs: 12_000, + }); + await valid.reporter + .wrap( + phase("gateway", async (context) => ({ + context, + result: advanceTo("provider_selection"), + })), + ) + .run("ctx"); + expect(valid.state.timerIntervalMs).toBe(12_000); + }); + + it("provides a friendly label for every non-terminal state", () => { + for (const [state, label] of Object.entries(ONBOARD_PHASE_LABELS)) { + expect(label.trim().length, state).toBeGreaterThan(0); + } + }); +}); diff --git a/src/lib/onboard/machine/phase-progress.ts b/src/lib/onboard/machine/phase-progress.ts new file mode 100644 index 0000000000..2b3af89864 --- /dev/null +++ b/src/lib/onboard/machine/phase-progress.ts @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OnboardSequencePhase } from "./sequence-runner"; +import type { OnboardNonTerminalMachineState } from "./types"; + +export const ONBOARD_PHASE_LABELS: Readonly> = { + init: "Initialization", + preflight: "Preflight checks", + gateway: "Gateway startup", + provider_selection: "Provider selection", + inference: "Inference setup", + sandbox: "Sandbox creation", + agent_setup: "Agent setup", + openclaw: "OpenClaw setup", + policies: "Network policies", + finalizing: "Finalization", + post_verify: "Verification", +}; + +const HEARTBEAT_PHASE_STATES: ReadonlySet = new Set([ + "gateway", + "inference", + "sandbox", + "agent_setup", + "openclaw", + "finalizing", + "post_verify", +]); +const INTERACTIVE_PHASE_STATES: ReadonlySet = new Set([ + "provider_selection", + "policies", +]); +const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000; + +export interface PhaseProgressTimer { + unref?(): void; +} + +export interface PhaseProgressOptions { + enabled?: boolean; + interactive?: boolean; + heartbeatIntervalMs?: number; + logLine?: (line: string) => void; + now?: () => number; + setTimer?: (callback: () => void, intervalMs: number) => PhaseProgressTimer; + clearTimer?: (timer: PhaseProgressTimer) => void; +} + +export interface PhaseProgressReporter { + wrap(phase: OnboardSequencePhase): OnboardSequencePhase; +} + +/** Add bounded progress output around one onboarding phase without shared state. */ +export function createPhaseProgressReporter( + options: PhaseProgressOptions = {}, +): PhaseProgressReporter { + const interactive = options.interactive ?? process.env.NEMOCLAW_NON_INTERACTIVE !== "1"; + const heartbeatStates = interactive + ? HEARTBEAT_PHASE_STATES + : new Set([...HEARTBEAT_PHASE_STATES, ...INTERACTIVE_PHASE_STATES]); + const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS; + const logLine = options.logLine ?? console.log; + const now = options.now ?? Date.now; + const setTimer = + options.setTimer ?? + ((callback: () => void, intervalMs: number) => setInterval(callback, intervalMs)); + const clearTimer = + options.clearTimer ?? ((timer: PhaseProgressTimer) => clearInterval(timer as NodeJS.Timeout)); + const enabled = options.enabled ?? true; + + return { + wrap(phase: OnboardSequencePhase): OnboardSequencePhase { + if (!enabled || !heartbeatStates.has(phase.state)) return phase; + return { + state: phase.state, + async run(context) { + const label = ONBOARD_PHASE_LABELS[phase.state]; + const startedAt = now(); + const timer = setTimer(() => { + const elapsedSeconds = Math.max(0, Math.round((now() - startedAt) / 1000)); + try { + logLine(` ⏳ Still working on ${label}… (${elapsedSeconds}s elapsed)`); + } catch { + // Progress output must never interrupt onboarding. + } + }, heartbeatIntervalMs); + timer.unref?.(); + try { + return await phase.run(context); + } finally { + try { + clearTimer(timer); + } catch { + // The timer may already have been cleared during shutdown. + } + } + }, + }; + }, + }; +} diff --git a/src/lib/onboard/machine/sequence-runner.test.ts b/src/lib/onboard/machine/sequence-runner.test.ts index 6791168566..a09c52ecd6 100644 --- a/src/lib/onboard/machine/sequence-runner.test.ts +++ b/src/lib/onboard/machine/sequence-runner.test.ts @@ -7,17 +7,17 @@ import { createSession, filterSafeUpdates, normalizeSession, - sanitizeFailure, type Session, type SessionUpdates, + sanitizeFailure, } from "../../state/onboard-session"; import { advanceTo, branchTo, completeOnboardMachine, retryTo } from "./result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./runtime"; import { buildOnboardSequenceHandlers, DuplicateOnboardSequencePhaseError, - runOnboardSequenceWithRunner, type OnboardSequencePhase, + runOnboardSequenceWithRunner, } from "./sequence-runner"; interface SequenceContext { @@ -85,6 +85,7 @@ function phase( describe("onboard sequence runner", () => { it("runs sequence phases through the strict FSM runner", async () => { + const wrappedStates: string[] = []; const phases: OnboardSequencePhase[] = [ phase("init", (context) => ({ context: { ...context, log: [...context.log, "init"] }, @@ -142,6 +143,12 @@ describe("onboard sequence runner", () => { context: { attempt: 0, log: [] }, runtime: createRuntime(), phases, + phaseProgress: { + wrap: (candidate) => { + wrappedStates.push(candidate.state); + return candidate; + }, + }, }); expect(result.session).toMatchObject({ @@ -164,6 +171,7 @@ describe("onboard sequence runner", () => { "post_verify", ], }); + expect(wrappedStates).toEqual(phases.map((candidate) => candidate.state)); }); it("passes custom sequence ownership through to the runner", async () => { diff --git a/src/lib/onboard/machine/sequence-runner.ts b/src/lib/onboard/machine/sequence-runner.ts index b2d71a18e6..ac64e859a3 100644 --- a/src/lib/onboard/machine/sequence-runner.ts +++ b/src/lib/onboard/machine/sequence-runner.ts @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createPhaseProgressReporter, type PhaseProgressReporter } from "./phase-progress"; import type { OnboardMachineRunnerOptions, OnboardStateHandlerResult } from "./runner"; import { - runOnboardMachine, type OnboardMachineRunnerRuntime, type OnboardStateHandlers, + runOnboardMachine, } from "./runner"; import type { OnboardNonTerminalMachineState } from "./types"; @@ -28,6 +29,11 @@ export interface OnboardSequenceRunnerOptions { maxTransitions?: OnboardMachineRunnerOptions["maxTransitions"]; sequenceOwnership?: OnboardMachineRunnerOptions["sequenceOwnership"]; stopStates?: OnboardMachineRunnerOptions["stopStates"]; + /** + * Phase-level progress reporter. Defaults to bounded heartbeats and is + * injectable for focused tests. + */ + phaseProgress?: PhaseProgressReporter; } export class DuplicateOnboardSequencePhaseError extends Error { @@ -43,9 +49,11 @@ export class DuplicateOnboardSequencePhaseError extends Error { export function buildOnboardSequenceHandlers( phases: readonly OnboardSequencePhase[], setPendingContext: (context: Context) => void, + phaseProgress: PhaseProgressReporter = createPhaseProgressReporter(), ): OnboardStateHandlers { const handlers: OnboardStateHandlers = {}; - for (const phase of phases) { + for (const rawPhase of phases) { + const phase = phaseProgress.wrap(rawPhase); if (handlers[phase.state]) throw new DuplicateOnboardSequencePhaseError(phase.state); handlers[phase.state] = async (context) => { const phaseResult = await phase.run(context); @@ -71,6 +79,7 @@ export async function runOnboardSequenceWithRunner({ maxTransitions, sequenceOwnership, stopStates, + phaseProgress, }: OnboardSequenceRunnerOptions) { let pendingContext = initialContext; return runOnboardMachine({ @@ -79,9 +88,13 @@ export async function runOnboardSequenceWithRunner({ maxTransitions, sequenceOwnership, stopStates, - handlers: buildOnboardSequenceHandlers(phases, (context) => { - pendingContext = context; - }), + handlers: buildOnboardSequenceHandlers( + phases, + (context) => { + pendingContext = context; + }, + phaseProgress, + ), updateContext: () => pendingContext, }); } diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index dda31ff4c2..1a890c7a16 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -9,7 +9,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createOpenshellCliHelpers } from "./openshell-cli"; -import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; +import { + prepareSandboxCreateLaunch, + prepareSandboxCreateLaunchWithPrebuild, +} from "./sandbox-create-launch"; const disabledHermesDashboardState = { config: null, enabled: false }; @@ -259,3 +262,72 @@ describe("prepareSandboxCreateLaunch", () => { expect(result.envArgs.some((arg) => arg.startsWith("NEMOCLAW_SANDBOX_NAME="))).toBe(false); }); }); + +describe("prepareSandboxCreateLaunchWithPrebuild", () => { + it("hands the build-qualified image to the canonical launch renderer", async () => { + const buildImage = vi.fn(async () => 0); + const result = await prepareSandboxCreateLaunchWithPrebuild({ + agent: null, + chatUiUrl: "", + createArgs: ["--from", "/tmp/build/Dockerfile", "--name", "demo"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + sandboxName: "demo", + buildEnv: () => ({}), + prebuild: { + buildCtx: "/tmp/build", + buildId: "build-123", + dockerDriverGateway: true, + env: { NEMOCLAW_SANDBOX_PREBUILD: "1" }, + buildImage, + log: vi.fn(), + }, + }); + + expect(result.prebuild).toEqual({ + createArgs: ["--from", "nemoclaw-sandbox-local:demo-build-123", "--name", "demo"], + imageRef: "nemoclaw-sandbox-local:demo-build-123", + }); + expect(result.createCommand).toContain( + "sandbox create --from nemoclaw-sandbox-local:demo-build-123 --name demo", + ); + expect(buildImage).toHaveBeenCalledOnce(); + }); + + it("renders the original Dockerfile after a local build failure", async () => { + const result = await prepareSandboxCreateLaunchWithPrebuild({ + agent: null, + chatUiUrl: "", + createArgs: ["--from", "/tmp/build/Dockerfile", "--name", "demo"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + sandboxName: "demo", + buildEnv: () => ({}), + prebuild: { + buildCtx: "/tmp/build", + buildId: "build-123", + dockerDriverGateway: true, + env: { NEMOCLAW_SANDBOX_PREBUILD: "1" }, + buildImage: async () => 1, + log: vi.fn(), + }, + }); + + expect(result.prebuild).toEqual({ + createArgs: ["--from", "/tmp/build/Dockerfile", "--name", "demo"], + imageRef: null, + }); + expect(result.createCommand).toContain( + "sandbox create --from /tmp/build/Dockerfile --name demo", + ); + expect(result.createCommand).not.toContain("nemoclaw-sandbox-local"); + }); +}); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 73203db0f0..24db00e38d 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -10,6 +10,11 @@ import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; +import { + prebuildSandboxImageIfEligible, + type SandboxPrebuildInput, + type SandboxPrebuildResult, +} from "./sandbox-prebuild"; type OpenshellShellCommand = (args: string[]) => string; @@ -35,6 +40,15 @@ export interface SandboxCreateLaunch { sandboxStartupCommand: string[]; } +export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { + sandboxName: string; + prebuild: Omit; +} + +export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { + prebuild: SandboxPrebuildResult; +} + export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): SandboxCreateLaunch { const env = input.env ?? process.env; const manageDashboard = input.manageDashboard ?? true; @@ -111,3 +125,19 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San sandboxStartupCommand, }; } + +/** Coordinate the optional local image build with the canonical launch renderer. */ +export async function prepareSandboxCreateLaunchWithPrebuild( + input: SandboxCreateLaunchWithPrebuildInput, +): Promise { + const { prebuild: prebuildInput, ...launchInput } = input; + const prebuild = await prebuildSandboxImageIfEligible({ + ...prebuildInput, + createArgs: input.createArgs, + sandboxName: input.sandboxName, + }); + return { + ...prepareSandboxCreateLaunch({ ...launchInput, createArgs: prebuild.createArgs }), + prebuild, + }; +} diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts new file mode 100644 index 0000000000..dee44f0f26 --- /dev/null +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + dockerBuildSubprocessEnv, + prebuildSandboxImageIfEligible, + resolveSandboxPrebuildEnabled, + sandboxLocalImageRef, +} from "./sandbox-prebuild"; + +const BUILD_CONTEXT = "/tmp/nemoclaw-build-abc"; +const BUILD_ID = "1234567890"; +const DOCKERFILE = `${BUILD_CONTEXT}/Dockerfile`; +const CREATE_ARGS = ["--from", DOCKERFILE, "--name", "alpha"]; + +describe("sandbox BuildKit prebuild", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("keeps Docker runtime settings while dropping secrets and control-plane state", () => { + vi.stubEnv("PATH", "/usr/bin"); + vi.stubEnv("HOME", "/home/user"); + vi.stubEnv("DOCKER_HOST", "unix:///var/run/docker.sock"); + vi.stubEnv("DOCKER_CONFIG", "/home/user/.docker-ci"); + vi.stubEnv("DOCKER_CONTEXT", "remote-builder"); + vi.stubEnv("XDG_CONFIG_HOME", "/home/user/.config"); + vi.stubEnv("HTTPS_PROXY", "http://proxy:8080"); + vi.stubEnv("NVIDIA_INFERENCE_API_KEY", "secret"); + vi.stubEnv("GITHUB_TOKEN", "secret"); + vi.stubEnv("KUBECONFIG", "/home/user/.kube/config"); + vi.stubEnv("SSH_AUTH_SOCK", "/tmp/agent.sock"); + vi.stubEnv("OPENSHELL_GATEWAY", "nemoclaw"); + vi.stubEnv("GRPC_VERBOSITY", "debug"); + + const env = dockerBuildSubprocessEnv(); + + expect(env).toMatchObject({ + PATH: "/usr/bin", + HOME: "/home/user", + DOCKER_HOST: "unix:///var/run/docker.sock", + DOCKER_CONFIG: "/home/user/.docker-ci", + DOCKER_CONTEXT: "remote-builder", + XDG_CONFIG_HOME: "/home/user/.config", + HTTPS_PROXY: "http://proxy:8080", + }); + for (const key of [ + "NVIDIA_INFERENCE_API_KEY", + "GITHUB_TOKEN", + "KUBECONFIG", + "SSH_AUTH_SOCK", + "OPENSHELL_GATEWAY", + "GRPC_VERBOSITY", + ]) { + expect(env[key], key).toBeUndefined(); + } + }); + + it("never enables a local-image handoff for a remote gateway", () => { + expect(resolveSandboxPrebuildEnabled({}, false)).toBe(false); + expect(resolveSandboxPrebuildEnabled({ NEMOCLAW_SANDBOX_PREBUILD: "1" }, false)).toBe(false); + }); + + it("defaults on locally, honors opt-out, and requires opt-in under tests", () => { + expect(resolveSandboxPrebuildEnabled({}, true)).toBe(true); + expect(resolveSandboxPrebuildEnabled({ NEMOCLAW_SANDBOX_PREBUILD: "0" }, true)).toBe(false); + expect(resolveSandboxPrebuildEnabled({ VITEST: "true" }, true)).toBe(false); + expect( + resolveSandboxPrebuildEnabled({ VITEST: "true", NEMOCLAW_SANDBOX_PREBUILD: "1" }, true), + ).toBe(true); + }); + + it("derives a build-unique local image tag", () => { + const imageRef = sandboxLocalImageRef("My Bot/2!", BUILD_ID); + expect(imageRef).toBe("nemoclaw-sandbox-local:my-bot-2--1234567890"); + expect(sandboxLocalImageRef("My Bot/2!", "next-build")).not.toBe(imageRef); + expect(sandboxLocalImageRef("a".repeat(128), "next-build")).not.toBe( + sandboxLocalImageRef("a".repeat(128), "other-build"), + ); + }); + + it("skips the build when create arguments do not use the staged Dockerfile", async () => { + const buildImage = vi.fn(async () => 0); + await expect( + prebuildSandboxImageIfEligible({ + buildCtx: BUILD_CONTEXT, + buildId: BUILD_ID, + createArgs: ["--from", "/other/Dockerfile"], + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + }), + ).resolves.toEqual({ createArgs: ["--from", "/other/Dockerfile"], imageRef: null }); + expect(buildImage).not.toHaveBeenCalled(); + }); + + it("uses the argv-based Docker helper and returns the local image on success", async () => { + const buildImage = vi.fn(async () => 0); + const result = await prebuildSandboxImageIfEligible({ + buildCtx: BUILD_CONTEXT, + buildId: BUILD_ID, + createArgs: CREATE_ARGS, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + log: () => {}, + }); + + expect(buildImage).toHaveBeenCalledWith( + [ + "build", + "--progress=plain", + "-t", + "nemoclaw-sandbox-local:alpha-1234567890", + "-f", + DOCKERFILE, + BUILD_CONTEXT, + ], + expect.objectContaining({ + env: expect.objectContaining({ DOCKER_BUILDKIT: "1" }), + stdio: "inherit", + }), + ); + expect(result).toEqual({ + createArgs: ["--from", "nemoclaw-sandbox-local:alpha-1234567890", "--name", "alpha"], + imageRef: "nemoclaw-sandbox-local:alpha-1234567890", + }); + }); + + it.each([ + ["nonzero result", async () => 1], + ["missing exit status", async () => null], + ])("falls back to OpenShell after a %s", async (_label, buildImage) => { + const result = await prebuildSandboxImageIfEligible({ + buildCtx: BUILD_CONTEXT, + buildId: BUILD_ID, + createArgs: CREATE_ARGS, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + log: () => {}, + }); + expect(result).toEqual({ createArgs: CREATE_ARGS, imageRef: null }); + }); + + it("falls back to OpenShell when the Docker helper throws", async () => { + const result = await prebuildSandboxImageIfEligible({ + buildCtx: BUILD_CONTEXT, + buildId: BUILD_ID, + createArgs: CREATE_ARGS, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage: async () => { + throw new Error("unavailable"); + }, + log: () => {}, + }); + expect(result).toEqual({ createArgs: CREATE_ARGS, imageRef: null }); + }); +}); diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts new file mode 100644 index 0000000000..9123502b8a --- /dev/null +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerSpawn } from "../adapters/docker/exec"; +import { buildSubprocessEnv } from "../subprocess-env"; + +const TRUTHY_FLAG_VALUES = new Set(["1", "true", "yes", "on"]); +const FALSY_FLAG_VALUES = new Set(["0", "false", "no", "off"]); +const LOCAL_IMAGE_REPO = "nemoclaw-sandbox-local"; +const DOCKER_ENV_NAMES = [ + "DOCKER_API_VERSION", + "DOCKER_CERT_PATH", + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_TLS_VERIFY", +] as const; + +export interface SandboxPrebuildInput { + buildCtx: string; + buildId: string; + createArgs: readonly string[]; + sandboxName: string; + dockerDriverGateway: boolean; + env?: NodeJS.ProcessEnv; + buildImage?: ( + args: readonly string[], + options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, + ) => Promise; + log?: (message: string) => void; +} + +export interface SandboxPrebuildResult { + createArgs: string[]; + imageRef: string | null; +} + +/** Restrict the host Docker build to environment values used by Docker itself. */ +export function dockerBuildSubprocessEnv(): Record { + const env = buildSubprocessEnv(); + for (const key of DOCKER_ENV_NAMES) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + for (const key of Object.keys(env)) { + if ( + key === "KUBECONFIG" || + key === "SSH_AUTH_SOCK" || + key === "RUST_LOG" || + key === "RUST_BACKTRACE" || + key.startsWith("OPENSHELL_") || + key.startsWith("GRPC_") + ) { + delete env[key]; + } + } + return env; +} + +export function resolveSandboxPrebuildEnabled( + env: NodeJS.ProcessEnv, + dockerDriverGateway: boolean, +): boolean { + // A registry-less local image is never visible to k3s or remote gateways. + // Keep this invariant ahead of every environment override. + if (!dockerDriverGateway) return false; + + const override = String(env.NEMOCLAW_SANDBOX_PREBUILD ?? "") + .trim() + .toLowerCase(); + if (FALSY_FLAG_VALUES.has(override)) return false; + if (TRUTHY_FLAG_VALUES.has(override)) return true; + return !env.VITEST && env.NODE_ENV !== "test"; +} + +export function sandboxLocalImageRef(sandboxName: string, buildId: string): string { + const sanitize = (value: string) => + value + .toLowerCase() + .replace(/[^a-z0-9_.-]/g, "-") + .replace(/^[-.]+/, ""); + const buildPart = sanitize(buildId).slice(-32) || "build"; + const namePart = sanitize(sandboxName).slice(0, 127 - buildPart.length) || "sandbox"; + return `${LOCAL_IMAGE_REPO}:${namePart}-${buildPart}`; +} + +/** + * Build the already-staged sandbox context with BuildKit on the shared local + * Docker daemon. Any failure preserves the original OpenShell build path. + * Remove this bridge once OpenShell uses BuildKit for this local-driver path; + * extraction and observable retirement criteria are tracked by #6258. + */ +export async function prebuildSandboxImageIfEligible( + input: SandboxPrebuildInput, +): Promise { + const createArgs = [...input.createArgs]; + const env = input.env ?? process.env; + if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { + return { createArgs, imageRef: null }; + } + const fromIndex = createArgs.indexOf("--from"); + if (fromIndex < 0 || createArgs[fromIndex + 1] !== `${input.buildCtx}/Dockerfile`) { + return { createArgs, imageRef: null }; + } + + const log = input.log ?? console.log; + const imageRef = sandboxLocalImageRef(input.sandboxName, input.buildId); + const buildImage = + input.buildImage ?? + ((args, options) => + new Promise((resolve, reject) => { + const child = dockerSpawn(args, { ...options, shell: false }); + child.once("error", reject); + child.once("close", resolve); + })); + log(" Building sandbox image with BuildKit (skips the slower in-gateway builder)..."); + + let status: number | null; + try { + status = await buildImage( + [ + "build", + "--progress=plain", + "-t", + imageRef, + "-f", + `${input.buildCtx}/Dockerfile`, + input.buildCtx, + ], + { + env: { ...dockerBuildSubprocessEnv(), DOCKER_BUILDKIT: "1" }, + stdio: "inherit", + }, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); + return { createArgs, imageRef: null }; + } + + if (status !== 0) { + const detail = status === null ? " without an exit status" : ` (exit ${status})`; + log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); + return { createArgs, imageRef: null }; + } + + createArgs[fromIndex + 1] = imageRef; + return { + createArgs, + imageRef, + }; +} diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index de68cddfb1..cce6a3e40b 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -5,10 +5,12 @@ import { describe, expect, it, vi } from "vitest"; import { getSandboxFailurePhase, isSandboxReady } from "../state/gateway"; import { + createSandboxReadyWaiter, formatCreatedSandboxReadinessFailureMessage, getSandboxReadyErrorDebouncePolls, SANDBOX_READY_ERROR_DEBOUNCE_ENV, waitForCreatedSandboxReadyWithTrace, + waitForSandboxReadyWithTrace, } from "./sandbox-readiness-tracing"; const NAME = "my-sandbox"; @@ -20,6 +22,65 @@ function replay(outputs: readonly string[]) { return { runCaptureOpenshell, sleep, polls: () => i }; } +describe("createSandboxReadyWaiter", () => { + it("uses the bounded Docker-driver polling defaults without a final delay", () => { + const runCaptureOpenshell = vi.fn(() => `${NAME} Provisioning`); + const sleep = vi.fn(); + const waitForSandboxReady = createSandboxReadyWaiter({ + runCaptureOpenshell, + isSandboxReady, + isLinuxDockerDriverGatewayEnabled: () => true, + sleep, + }); + + expect(waitForSandboxReady(NAME, 2, 3)).toBe(false); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(3); + }); + + it("preserves the legacy Kubernetes pod fallback and final delay", () => { + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce(`${NAME} Provisioning`) + .mockReturnValueOnce("Pending"); + const sleep = vi.fn(); + const waitForSandboxReady = createSandboxReadyWaiter({ + runCaptureOpenshell, + isSandboxReady, + isLinuxDockerDriverGatewayEnabled: () => false, + sleep, + }); + + expect(waitForSandboxReady(NAME, 1, 2)).toBe(false); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(runCaptureOpenshell.mock.calls[1]?.[0]).toContain("kubectl"); + expect(sleep).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(2); + }); + + it("keeps the traced waiter free of the legacy final delay", () => { + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce(`${NAME} Provisioning`) + .mockReturnValueOnce("Pending"); + const sleep = vi.fn(); + + expect( + waitForSandboxReadyWithTrace({ + sandboxName: NAME, + attempts: 1, + delaySeconds: 2, + runCaptureOpenshell, + isSandboxReady, + isLinuxDockerDriverGatewayEnabled: () => false, + sleep, + }), + ).toBe(false); + expect(sleep).not.toHaveBeenCalled(); + }); +}); + describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { it("fast-fails on the first Error poll when the debounce is opted out (K=1)", () => { const { runCaptureOpenshell, sleep } = replay([ diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index aeef5a08ad..3202016104 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -76,15 +76,25 @@ export type CreatedSandboxReadinessResult = | { ready: false; reason: "terminal_failure_phase"; failurePhase: string | null } | { ready: false; reason: "timeout"; failurePhase: null }; -export function waitForSandboxReadyWithTrace(options: { - sandboxName: string; - attempts: number; - delaySeconds: number; +export interface SandboxReadyWaitDeps { runCaptureOpenshell: RunCaptureOpenshell; isSandboxReady: (output: string, sandboxName: string) => boolean; isLinuxDockerDriverGatewayEnabled: () => boolean; sleep: (seconds: number) => void; -}): boolean { +} + +export interface SandboxReadyWaitOptions extends SandboxReadyWaitDeps { + sandboxName: string; + attempts: number; + delaySeconds: number; +} + +function pollSandboxReady( + options: SandboxReadyWaitOptions & { + sleepAfterFinalPodPoll?: boolean; + trace?: (event: string, attributes: Record) => void; + }, +): boolean { const { sandboxName, attempts, @@ -94,45 +104,64 @@ export function waitForSandboxReadyWithTrace(options: { isLinuxDockerDriverGatewayEnabled, sleep, } = options; - return withSandboxReadinessTrace(sandboxName, { attempts, delay_seconds: delaySeconds }, () => { - for (let i = 0; i < attempts; i += 1) { - const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - if (isSandboxReady(list, sandboxName)) { - addTraceEvent("ready", { attempt: i + 1, source: "sandbox_list" }); - return true; - } + for (let i = 0; i < attempts; i += 1) { + const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); + if (isSandboxReady(list, sandboxName)) { + options.trace?.("ready", { attempt: i + 1, source: "sandbox_list" }); + return true; + } - // Package-managed OpenShell gateways report readiness through - // `sandbox list`; legacy Kubernetes gateways may still expose pod state. - if (isLinuxDockerDriverGatewayEnabled()) { - if (i < attempts - 1) sleep(delaySeconds); - continue; - } - const podPhase = runCaptureOpenshell( - [ - "doctor", - "exec", - "--", - "kubectl", - "-n", - "openshell", - "get", - "pod", - sandboxName, - "-o", - "jsonpath={.status.phase}", - ], - { ignoreError: true }, - ); - if (podPhase === "Running") { - addTraceEvent("ready", { attempt: i + 1, source: "pod_phase" }); - return true; - } + // Package-managed OpenShell gateways report readiness through + // `sandbox list`; legacy Kubernetes gateways may still expose pod state. + if (isLinuxDockerDriverGatewayEnabled()) { if (i < attempts - 1) sleep(delaySeconds); + continue; } - addTraceEvent("not_ready", { attempts }); - return false; - }); + const podPhase = runCaptureOpenshell( + [ + "doctor", + "exec", + "--", + "kubectl", + "-n", + "openshell", + "get", + "pod", + sandboxName, + "-o", + "jsonpath={.status.phase}", + ], + { ignoreError: true }, + ); + if (podPhase === "Running") { + options.trace?.("ready", { attempt: i + 1, source: "pod_phase" }); + return true; + } + if (i < attempts - 1 || options.sleepAfterFinalPodPoll) sleep(delaySeconds); + } + options.trace?.("not_ready", { attempts }); + return false; +} + +export function waitForSandboxReadyWithTrace(options: SandboxReadyWaitOptions): boolean { + return withSandboxReadinessTrace( + options.sandboxName, + { attempts: options.attempts, delay_seconds: options.delaySeconds }, + () => pollSandboxReady({ ...options, trace: addTraceEvent }), + ); +} + +export function createSandboxReadyWaiter( + deps: SandboxReadyWaitDeps, +): (sandboxName: string, attempts?: number, delaySeconds?: number) => boolean { + return (sandboxName, attempts = 10, delaySeconds = 2) => + pollSandboxReady({ + sandboxName, + attempts, + delaySeconds, + ...deps, + sleepAfterFinalPodPoll: true, + }); } export function waitForCreatedSandboxReadyWithTrace(options: { diff --git a/test/e2e-release-gate-workflow.test.ts b/test/e2e-release-gate-workflow.test.ts index efa8d78dd3..2ae62c7684 100644 --- a/test/e2e-release-gate-workflow.test.ts +++ b/test/e2e-release-gate-workflow.test.ts @@ -17,6 +17,11 @@ describe("release gate workflow resource contracts", () => { expect(fullJob.needs).toBe("generate-matrix"); expect(fullJob.if).not.toContain("always()"); expect(fullJob.if).toContain(",full-e2e,"); + expect( + fullJob.steps?.find((step) => step.name === "Run full-e2e live Vitest test")?.run, + ).toMatch( + /full-e2e\.test\.ts[\s\S]*npx vitest run --project e2e-live[\s\S]*onboard-progress-budget\.test\.ts/, + ); expect(tuiJob.needs).toBe("generate-matrix"); expect(tuiJob.if).not.toContain("always()"); expect(tuiJob.if).toContain(",openclaw-tui-chat-correlation,"); diff --git a/test/e2e/fixtures/shell-probe.ts b/test/e2e/fixtures/shell-probe.ts index bc11043478..f6efe7842f 100644 --- a/test/e2e/fixtures/shell-probe.ts +++ b/test/e2e/fixtures/shell-probe.ts @@ -26,6 +26,13 @@ export interface ShellProbeRunOptions { killGraceMs?: number; artifactName?: string; redactionValues?: string[]; + /** Timestamp-only output observer; chunk contents never cross this boundary. */ + onOutput?: (event: ShellProbeOutputEvent) => void; +} + +export interface ShellProbeOutputEvent { + stream: "stdout" | "stderr"; + atMs: number; } export type { TrustedShellCommand, TrustedShellCommandInput } from "./shell/trusted-command.ts"; @@ -131,9 +138,19 @@ export class ShellProbe { signal: this.signal, onStdout: (chunk) => { stdout += chunk; + try { + options.onOutput?.({ stream: "stdout", atMs: Date.now() }); + } catch { + // Test instrumentation must not change command execution. + } }, onStderr: (chunk) => { stderr += chunk; + try { + options.onOutput?.({ stream: "stderr", atMs: Date.now() }); + } catch { + // Test instrumentation must not change command execution. + } }, }); diff --git a/test/e2e/live/onboard-progress-budget.test.ts b/test/e2e/live/onboard-progress-budget.test.ts new file mode 100644 index 0000000000..99aa1b857b --- /dev/null +++ b/test/e2e/live/onboard-progress-budget.test.ts @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// +// Live acceptance test for issue #6002. It measures the issue's actual +// acceptance path — onboard step [1/8] through the first agent response — and +// asserts a real worktree-CLI onboard: +// 1. never leaves a wait-heavy phase silent longer than the 60s guarantee +// (proved from timestamped stdout/stderr chunks), and +// 2. builds the sandbox image with BuildKit (the prebuild speed path), and +// 3. reaches the first agent response (a headless `openclaw agent` turn that +// returns a real hosted-inference reply), and +// 4. does all of that within the ≤3-minute budget (NEMOCLAW_E2E_ONBOARD_BUDGET_SECS). +// +// Uses real hosted inference (NVIDIA_INFERENCE_API_KEY) because a genuine first +// response requires a real LLM turn — a stub endpoint completes onboarding's +// inference smoke but cannot drive a full agent turn. Opt-in via +// NEMOCLAW_RUN_LIVE_E2E=1; requires the hosted-inference key. + +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; +import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeOutputEvent, ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { extractOpenClawAgentText } from "./agent-turn-latency-helpers.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const HOSTED_INFERENCE_SECRET = "NVIDIA_INFERENCE_API_KEY"; +const SANDBOX_NAME = process.env.NEMOCLAW_E2E_PROGRESS_SANDBOX ?? "e2e-progress-budget"; +// Timeout env vars are named *_SECS because their values are seconds (×1000 +// below), matching their unit. +const ONBOARD_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_ONBOARD_TIMEOUT_SECS ?? 1_200) * 1_000; +const FIRST_TURN_TIMEOUT_MS = + Number(process.env.NEMOCLAW_E2E_FIRST_TURN_TIMEOUT_SECS ?? 240) * 1_000; +// Budget for the whole [1/8]-to-first-response path. Defaults to the issue's +// ≤3-minute goal (180s); constrained / cold-cache runners can raise +// NEMOCLAW_E2E_ONBOARD_BUDGET_SECS. +const BUDGET_SECS = Number(process.env.NEMOCLAW_E2E_ONBOARD_BUDGET_SECS ?? 180); +// The issue's guarantee: no onboarding phase stays silent longer than this. +const MAX_SILENCE_SECS = Number(process.env.NEMOCLAW_E2E_MAX_SILENCE_SECS ?? 60); +const TEST_TIMEOUT_MS = 45 * 60_000; +// Gated at declaration (no in-body `if`): live E2E is explicitly opt-in. +const liveTest = shouldRunLiveE2E() ? test : test.skip; + +validateSandboxName(SANDBOX_NAME); + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + OPENSHELL_GATEWAY: "nemoclaw", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +function onboardEnv(apiKey: string): NodeJS.ProcessEnv { + return commandEnv({ + // NVIDIA Endpoints hosted inference (default non-interactive provider). + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_DASHBOARD_PORT: "", + CHAT_UI_URL: "", + NEMOCLAW_RECREATE_SANDBOX: "1", + // Force the BuildKit prebuild path on under the Vitest-hosted live test. + NEMOCLAW_SANDBOX_PREBUILD: "1", + }); +} + +async function ignoreCleanupError(run: () => Promise): Promise { + try { + await run(); + } catch { + // Best-effort cleanup; never mask the lifecycle assertions. + } +} + +async function cleanupProgressState(host: HostCliClient, sandbox: SandboxClient): Promise { + await ignoreCleanupError(() => + host.command(process.execPath, [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env: commandEnv(), + timeoutMs: 180_000, + }), + ); + await ignoreCleanupError(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + await ignoreCleanupError(() => + sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +liveTest( + "onboard [1/8] reaches a first response within 3 minutes without a 60-second output gap (#6002)", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const apiKey = secrets.required(HOSTED_INFERENCE_SECRET); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(docker.exitCode, resultText(docker)).toBe(0); + + cleanup.add("remove progress-budget sandbox and gateway", async () => { + await cleanupProgressState(host, sandbox); + }); + await cleanupProgressState(host, sandbox); + + // Starting before process spawn is a conservative upper bound for the + // issue's literal [1/8]-to-response budget; the output assertion below + // proves that the expected wizard anchor was actually reached. + const startedAt = Date.now(); + const outputEvents: ShellProbeOutputEvent[] = []; + const onboard: ShellProbeResult = await host.command( + process.execPath, + [CLI_ENTRYPOINT, "onboard", "--non-interactive", "--no-gpu"], + { + artifactName: "onboard-progress-budget", + env: onboardEnv(apiKey), + onOutput: (event) => outputEvents.push(event), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const onboardFinishedAt = Date.now(); + const onboardSecs = Math.round((onboardFinishedAt - startedAt) / 1000); + + // Strip ANSI so text assertions are colour-independent (ESC built from a + // char code so there is no control literal in source). + const ansiSgr = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g"); + const plain = resultText(onboard).replace(ansiSgr, ""); + const heartbeatCount = (plain.match(/Still working on /g) ?? []).length; + const usedBuildKitPrebuild = /Building sandbox image with BuildKit/.test(plain); + const classicBuildSteps = (plain.match(/Step \d+\/\d+ :/g) ?? []).length; + + const outputTimes = [startedAt, ...outputEvents.map((event) => event.atMs), onboardFinishedAt]; + const maxSilenceSecs = Math.ceil( + Math.max(...outputTimes.slice(1).map((atMs, index) => atMs - outputTimes[index])) / 1000, + ); + + expect(onboard.exitCode, plain).toBe(0); + expect(plain, "expected literal wizard step [1/8] in onboard output").toContain("[1/8]"); + // (2) BuildKit prebuild ran (the speed fix), not the classic in-gateway builder. + expect(usedBuildKitPrebuild, "expected the BuildKit prebuild to run").toBe(true); + expect(classicBuildSteps, "expected no classic per-instruction build steps").toBe(0); + // (1) Adjacent terminal output chunks never exceeded the 60-second + // guarantee. Heartbeats account for otherwise quiet phases. + expect( + maxSilenceSecs, + `longest silent gap ${maxSilenceSecs}s exceeds the ${MAX_SILENCE_SECS}s guarantee`, + ).toBeLessThanOrEqual(MAX_SILENCE_SECS); + // (3) First agent response: a real headless `openclaw agent` turn. This is + // the scriptable equivalent of the issue's first TUI message. + const turn = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + "openclaw agent --agent main --json --thinking off --session-id e2e-6002 " + + "-m 'Reply with a short acknowledgement.'", + ), + { + artifactName: "onboard-first-agent-turn", + env: commandEnv(), + redactionValues: [apiKey], + timeoutMs: FIRST_TURN_TIMEOUT_MS, + }, + ); + const totalMs = Date.now() - startedAt; + const totalSecs = Math.ceil(totalMs / 1000); + const turnText = resultText(turn); + // Parse the `--json` payload and measure the assistant reply text — a raw + // non-empty output could just be a JSON envelope / log noise, so it would + // not prove the agent actually returned content (CodeRabbit). + const assistantReply = extractOpenClawAgentText(turnText); + const responseChars = assistantReply.trim().length; + + await artifacts.writeJson("onboard-progress-budget.json", { + sandbox: SANDBOX_NAME, + onboardExitCode: onboard.exitCode, + firstTurnExitCode: turn.exitCode, + onboardSecs, + totalMs, + totalSecs, + budgetSecs: BUDGET_SECS, + heartbeatCount, + maxSilenceSecs, + maxSilenceBudgetSecs: MAX_SILENCE_SECS, + usedBuildKitPrebuild, + classicBuildSteps, + responseChars, + }); + + expect(turn.exitCode, turnText).toBe(0); + // A real, non-empty first response came back (not just a completed onboard). + expect( + responseChars, + `expected a non-empty first agent reply, got: ${turnText}`, + ).toBeGreaterThan(0); + + // (4) Process start is earlier than [1/8], so this is a stricter upper + // bound than the issue's [1/8]-to-first-response budget. + expect( + totalMs, + `[1/8]-to-first-response took ${totalSecs}s, over the ${BUDGET_SECS}s budget`, + ).toBeLessThanOrEqual(BUDGET_SECS * 1_000); + }, +); diff --git a/test/onboard-sandbox-name.test.ts b/test/onboard-sandbox-name.test.ts index 2d2f885d15..33dd9df3e8 100644 --- a/test/onboard-sandbox-name.test.ts +++ b/test/onboard-sandbox-name.test.ts @@ -172,7 +172,7 @@ const onboardModule = require(${onboardPath}); } catch (error) { exitCode = error.exitCode ?? null; process.stdout.write( - JSON.stringify({ completed: false, exitCode, lines, message: error.message }), + JSON.stringify({ completed: false, exitCode, lines, message: error.message, nonInteractiveEnv: process.env.NEMOCLAW_NON_INTERACTIVE }), ); } finally { console.error = originalError; @@ -188,12 +188,13 @@ const onboardModule = require(${onboardPath}); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", - env: { ...process.env, HOME: tmpDir }, + env: { ...process.env, HOME: tmpDir, NEMOCLAW_NON_INTERACTIVE: "preserve-me" }, }); assert.equal(result.status, 0, result.stderr); const payload = JSON.parse(result.stdout.trim()); assert.equal(payload.completed, false); assert.equal(payload.exitCode, 1); + assert.equal(payload.nonInteractiveEnv, "preserve-me"); assert.ok( payload.lines.some((line: string) => line.includes("Invalid sandbox name: 'MyAssistant'.")), `expected 'Invalid sandbox name' line, got ${JSON.stringify(payload.lines)}`,