diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 741961df18..5fc4c5aa17 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -161,6 +161,10 @@ If the installed OpenShell version falls outside this range, onboarding exits wi Build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. The entire parent directory of the specified file is used as the Docker build context, so any files your Dockerfile references (scripts, config, etc.) must live alongside it. +Onboarding skips common large directories (`node_modules`, `.git`, `.venv`, and `__pycache__`) while staging this context. +It also skips credential-style files and directories such as `.env*`, `.ssh/`, `.aws/`, `.netrc`, `.npmrc`, `secrets/`, `*.pem`, and `*.key`. +Other build outputs such as `dist/`, `target/`, or `build/` are still included. +If the staged context is larger than 100 MB, onboarding prints a warning before the Docker build starts. If the directory contains unreadable files (for example, Windows system files visible in WSL), onboarding exits with an error suggesting you move the Dockerfile to a dedicated directory. ```console @@ -168,6 +172,14 @@ $ nemoclaw onboard --from path/to/Dockerfile ``` The file can have any name; if it is not already named `Dockerfile`, onboard copies it to `Dockerfile` inside the staged build context automatically. +To create an isolated build context, create a dedicated directory that contains only the Dockerfile and the files it needs: + +```text +build-dir/ +├── Dockerfile +└── files-used-by-COPY/ +``` + All NemoClaw build arguments (`NEMOCLAW_MODEL`, `NEMOCLAW_PROVIDER_KEY`, `NEMOCLAW_INFERENCE_BASE_URL`, etc.) are injected as `ARG` overrides at build time, so declare them in your Dockerfile if you need to reference them. In non-interactive mode, the path can also be supplied via the `NEMOCLAW_FROM_DOCKERFILE` environment variable. diff --git a/src/lib/onboard-command.test.ts b/src/lib/onboard-command.test.ts index 394eb36953..b691e58a45 100644 --- a/src/lib/onboard-command.test.ts +++ b/src/lib/onboard-command.test.ts @@ -116,6 +116,9 @@ describe("onboard command", () => { expect(lines.join("\n")).toContain("Usage: nemoclaw onboard"); expect(lines.join("\n")).toContain("--from "); expect(lines.join("\n")).toContain("--name "); + expect(lines.join("\n")).toContain("Dockerfile's parent directory"); + expect(lines.join("\n")).toContain("node_modules, .git, .venv, __pycache__"); + expect(lines.join("\n")).toContain(".env*, .ssh, .aws"); expect(lines.join("\n")).toContain("--agent "); expect(lines.join("\n")).toContain("--dangerously-skip-permissions"); }); @@ -293,6 +296,27 @@ describe("onboard command", () => { expect(errors.join("\n")).toContain("--from path not found:"); }); + it("exits before onboarding when --from points to a directory", async () => { + const runOnboard = vi.fn(async () => {}); + const errors: string[] = []; + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-dir-")); + + await expect( + runOnboardCommand({ + args: ["--from", tmpDir], + noticeAcceptFlag: "--yes-i-accept-third-party-software", + noticeAcceptEnv: "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + env: {}, + runOnboard, + error: (message = "") => errors.push(message), + exit: exitWithPrefixedCode, + }), + ).rejects.toThrow("exit:1"); + + expect(runOnboard).not.toHaveBeenCalled(); + expect(errors.join("\n")).toContain("--from must point to a Dockerfile:"); + }); + it("exits with usage on unknown args", () => { const errors: string[] = []; expect(() => diff --git a/src/lib/onboard-command.ts b/src/lib/onboard-command.ts index 4e17d80e5a..3271f1c278 100644 --- a/src/lib/onboard-command.ts +++ b/src/lib/onboard-command.ts @@ -45,6 +45,13 @@ function onboardUsageLines(noticeAcceptFlag: string): string[] { return [ ` Usage: nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--from ] [--name ] [--agent ] [--control-ui-port ] [--dangerously-skip-permissions] [${noticeAcceptFlag}]`, "", + " --from uses the Dockerfile's parent directory as the Docker build context.", + " Put files referenced by COPY/ADD next to that Dockerfile, or move the Dockerfile into", + " a dedicated build directory to avoid sending unrelated files to Docker.", + " Common large directories are skipped: node_modules, .git, .venv, __pycache__.", + " Credential-style files and directories such as .env*, .ssh, .aws, .netrc, .npmrc, secrets/, *.pem, and *.key are also skipped.", + " Generated output directories such as dist/, build/, and target/ are still included.", + "", ]; } @@ -78,6 +85,10 @@ export function parseOnboardArgs( error(` --from path not found: ${resolvedFromDockerfile}`); exit(1); } + if (!fs.statSync(resolvedFromDockerfile).isFile()) { + error(` --from must point to a Dockerfile: ${resolvedFromDockerfile}`); + exit(1); + } fromDockerfile = requestedFromDockerfile; parsedArgs.splice(fromIdx, 2); } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 8d0b4392f0..b95efcc60d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -64,7 +64,10 @@ function requireValue(value: T | null | undefined, message: string): T { } return value; } -const { stageOptimizedSandboxBuildContext } = require("./sandbox-build-context"); +const { + collectBuildContextStats, + stageOptimizedSandboxBuildContext, +} = require("./sandbox-build-context"); const { buildSubprocessEnv } = require("./subprocess-env"); const { DASHBOARD_PORT, @@ -92,6 +95,60 @@ const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference } const onboardProviders = require("./onboard-providers"); +const CUSTOM_BUILD_CONTEXT_WARN_BYTES = 100_000_000; +const CUSTOM_BUILD_CONTEXT_IGNORES = new Set([ + "node_modules", + ".git", + ".venv", + "__pycache__", + ".aws", + ".credentials", + ".direnv", + ".netrc", + ".npmrc", + ".pypirc", + ".ssh", + "credentials.json", + "key.json", + "secrets", + "secrets.json", + "secrets.yaml", + "token.json", +]); + +function isIgnoredCustomBuildContextName(name: string): boolean { + const lowerName = name.toLowerCase(); + return ( + CUSTOM_BUILD_CONTEXT_IGNORES.has(lowerName) || + lowerName === ".env" || + lowerName === ".envrc" || + lowerName.startsWith(".env.") || + lowerName.endsWith(".key") || + lowerName.endsWith(".pem") || + lowerName.endsWith(".pfx") || + lowerName.endsWith(".p12") || + lowerName.endsWith(".jks") || + lowerName.endsWith(".keystore") || + lowerName.endsWith(".tfvars") || + lowerName.endsWith("_ecdsa") || + lowerName.endsWith("_ed25519") || + lowerName.endsWith("_rsa") || + (lowerName.startsWith("service-account") && lowerName.endsWith(".json")) + ); +} + +function shouldIncludeCustomBuildContextPath(src: string): boolean { + return !isIgnoredCustomBuildContextName(path.basename(src)); +} + +function isInsideIgnoredCustomBuildContextPath(src: string): boolean { + return path + .normalize(src) + .split(path.sep) + .filter(Boolean) + .some((part: string) => isIgnoredCustomBuildContextName(part)); +} + type RemoteProviderConfigEntry = { label: string; providerName: string; @@ -133,8 +190,16 @@ const { getEffectiveProviderName: (key: string | null | undefined) => string | null; getNonInteractiveProvider: () => string | null; getNonInteractiveModel: (providerKey: string) => string | null; - getSandboxInferenceConfig: (model: string, provider?: string | null, preferredInferenceApi?: string | null) => { - providerKey: string; primaryModelRef: string; inferenceBaseUrl: string; inferenceApi: string; inferenceCompat: LooseObject | null; + getSandboxInferenceConfig: ( + model: string, + provider?: string | null, + preferredInferenceApi?: string | null, + ) => { + providerKey: string; + primaryModelRef: string; + inferenceBaseUrl: string; + inferenceApi: string; + inferenceCompat: LooseObject | null; }; }; const { sleepSeconds } = require("./wait"); @@ -282,7 +347,11 @@ const BRAVE_SEARCH_HELP_URL = "https://brave.com/search/api/"; // Re-export shared JSON types under the names used throughout this module. // See src/lib/json-types.ts for the canonical definitions. -import type { JsonScalar as LooseScalar, JsonValue as LooseValue, JsonObject as LooseObject } from "./json-types"; +import type { + JsonScalar as LooseScalar, + JsonValue as LooseValue, + JsonObject as LooseObject, +} from "./json-types"; type OnboardOptions = { nonInteractive?: boolean; @@ -1745,7 +1814,6 @@ const { // nvcfFunctionNotFoundMessage — see validation import above. They live in // src/lib/validation.ts so they can be unit-tested independently. - async function validateOpenAiLikeSelection( label: string, endpointUrl: string, @@ -1938,7 +2006,6 @@ function getRequestedProviderHint(nonInteractive = isNonInteractive()) { } function getRequestedModelHint(nonInteractive = isNonInteractive()) { return onboardProviders.getRequestedModelHint(nonInteractive); - } function getResumeConfigConflicts( @@ -2382,25 +2449,19 @@ async function preflight(): Promise> { console.warn( " ⚠ Container DNS probe inconclusive: docker couldn't pull the busybox test image.", ); - console.warn( - " This usually means the docker daemon itself can't reach Docker Hub,", - ); + console.warn(" This usually means the docker daemon itself can't reach Docker Hub,"); console.warn( " but doesn't prove container DNS is broken — the sandbox build may still succeed.", ); } else { - console.warn( - ` ⚠ Container DNS probe inconclusive (reason: ${dns.reason ?? "unknown"}).`, - ); + console.warn(` ⚠ Container DNS probe inconclusive (reason: ${dns.reason ?? "unknown"}).`); } if (dns.details) { for (const line of String(dns.details).split("\n").slice(-3)) { if (line.trim()) console.warn(` ${line.trim()}`); } } - console.warn( - " Proceeding. If the sandbox build later hangs at `npm ci`, see issue #2101.", - ); + console.warn(" Proceeding. If the sandbox build later hangs at `npm ci`, see issue #2101."); } else { console.error(" ✗ DNS resolution from inside a docker container failed."); if (dns.details) { @@ -2410,18 +2471,10 @@ async function preflight(): Promise> { } console.error(""); { - console.error( - " The sandbox build runs `npm ci` inside a container and needs to resolve", - ); - console.error( - " registry.npmjs.org. On networks that block outbound UDP:53 to public DNS", - ); - console.error( - " (common in corporate environments that force DNS-over-TLS on the host),", - ); - console.error( - " the build appears to hang for ~15 minutes and then prints the cryptic", - ); + console.error(" The sandbox build runs `npm ci` inside a container and needs to resolve"); + console.error(" registry.npmjs.org. On networks that block outbound UDP:53 to public DNS"); + console.error(" (common in corporate environments that force DNS-over-TLS on the host),"); + console.error(" the build appears to hang for ~15 minutes and then prints the cryptic"); console.error(" `npm error Exit handler never called`. See issue #2101."); console.error(""); console.error(" Fix options:"); @@ -2470,9 +2523,7 @@ async function preflight(): Promise> { console.error(" 1. Make systemd-resolved reachable from containers (recommended):"); printLinuxFix(bridgeIp, bridgeNote); console.error(""); - console.error( - " 2. Configure an explicit UDP:53-capable DNS in /etc/docker/daemon.json", - ); + console.error(" 2. Configure an explicit UDP:53-capable DNS in /etc/docker/daemon.json"); console.error(" (ask your IT team for an internal DNS server IP)."); } else if (host.platform === "darwin") { // On macOS, branch by the detected runtime (host.runtime) so users get @@ -2481,9 +2532,7 @@ async function preflight(): Promise> { console.error(" Configure Colima's DNS (macOS):"); console.error(" colima stop"); console.error(" colima start --dns "); - console.error( - " (or edit ~/.colima/default/colima.yaml and `colima restart`)", - ); + console.error(" (or edit ~/.colima/default/colima.yaml and `colima restart`)"); } else if (host.runtime === "docker-desktop" || host.runtime === "docker") { console.error(" Configure Docker Desktop's DNS (macOS):"); console.error( @@ -2501,7 +2550,7 @@ async function preflight(): Promise> { console.error(" Configure your container runtime's DNS (macOS):"); console.error(" - Docker Desktop:"); console.error( - " { jq '. + {\"dns\":[\"\"]}' ~/.docker/daemon.json 2>/dev/null || echo '{\"dns\":[\"\"]}'; } > ~/.docker/daemon.json.new && mv ~/.docker/daemon.json.new ~/.docker/daemon.json", + ' { jq \'. + {"dns":[""]}\' ~/.docker/daemon.json 2>/dev/null || echo \'{"dns":[""]}\'; } > ~/.docker/daemon.json.new && mv ~/.docker/daemon.json.new ~/.docker/daemon.json', ); console.error(" osascript -e 'quit app \"Docker\"' && sleep 3 && open -a Docker"); console.error(" - Colima:"); @@ -2509,13 +2558,9 @@ async function preflight(): Promise> { console.error(" - Rancher Desktop / Podman: edit the runtime's DNS config"); console.error(" and restart it."); } - console.error( - " Ask your IT team for an internal DNS server IP that accepts UDP:53.", - ); + console.error(" Ask your IT team for an internal DNS server IP that accepts UDP:53."); } else if (host.platform === "win32" || host.isWsl) { - console.error( - " 1. Configure Docker Desktop's DNS (Windows / WSL via Docker Desktop):", - ); + console.error(" 1. Configure Docker Desktop's DNS (Windows / WSL via Docker Desktop):"); console.error( " Docker Desktop for Windows → Settings → Docker Engine — edit the JSON to add:", ); @@ -2540,9 +2585,7 @@ async function preflight(): Promise> { } printLinuxFix(wslBridgeIp || "172.17.0.1", wslBridgeNote); } else { - console.error( - " Configure your docker daemon to use a DNS server that accepts UDP:53.", - ); + console.error(" Configure your docker daemon to use a DNS server that accepts UDP:53."); console.error( ' Add { "dns": [""] } to your docker daemon.json and restart the daemon.', ); @@ -3404,13 +3447,18 @@ async function createSandbox( if (process.env.CHAT_UI_URL) { try { const u = new URL( - process.env.CHAT_UI_URL.includes("://") ? process.env.CHAT_UI_URL : `http://${process.env.CHAT_UI_URL}`, + process.env.CHAT_UI_URL.includes("://") + ? process.env.CHAT_UI_URL + : `http://${process.env.CHAT_UI_URL}`, ); const p = Number(u.port); if (p > 0) envPort = p; - } catch { /* malformed URL — ignore */ } + } catch { + /* malformed URL — ignore */ + } } - const preferredPort = controlUiPort ?? envPort ?? persistedPort ?? (agent ? agent.forwardPort : CONTROL_UI_PORT); + const preferredPort = + controlUiPort ?? envPort ?? persistedPort ?? (agent ? agent.forwardPort : CONTROL_UI_PORT); const earlyForwards = runCaptureOpenshell(["forward", "list"], { ignoreError: true }); const effectivePort = findAvailableDashboardPort(sandboxName, preferredPort, earlyForwards); if (effectivePort !== preferredPort) { @@ -3421,7 +3469,9 @@ async function createSandbox( let chatUiUrl: string; if (process.env.CHAT_UI_URL && controlUiPort == null) { const parsed = new URL( - process.env.CHAT_UI_URL.includes("://") ? process.env.CHAT_UI_URL : `http://${process.env.CHAT_UI_URL}`, + process.env.CHAT_UI_URL.includes("://") + ? process.env.CHAT_UI_URL + : `http://${process.env.CHAT_UI_URL}`, ); parsed.port = String(effectivePort); chatUiUrl = parsed.toString().replace(/\/$/, ""); @@ -3714,29 +3764,66 @@ async function createSandbox( // in env args, so it must not persist in /tmp after a failed sandbox create. // run() calls process.exit() on failure (bypassing normal control flow), so // we register a process 'exit' handler to guarantee cleanup in all cases. - let buildCtx, stagedDockerfile; + let buildCtx: string, stagedDockerfile: string; if (fromDockerfile) { const fromResolved = path.resolve(fromDockerfile); if (!fs.existsSync(fromResolved)) { console.error(` Custom Dockerfile not found: ${fromResolved}`); process.exit(1); } + if (!fs.statSync(fromResolved).isFile()) { + console.error(` Custom Dockerfile path is not a file: ${fromResolved}`); + process.exit(1); + } + const buildContextDir = path.dirname(fromResolved); + if (isInsideIgnoredCustomBuildContextPath(buildContextDir)) { + console.error( + ` Custom Dockerfile is inside an ignored build-context path: ${buildContextDir}`, + ); + console.error(" Move your Dockerfile to a dedicated directory and retry."); + process.exit(1); + } + console.log(` Using custom Dockerfile: ${fromResolved}`); + console.log(` Docker build context: ${buildContextDir}`); + const buildContextStats = collectBuildContextStats( + buildContextDir, + shouldIncludeCustomBuildContextPath, + ); + if (buildContextStats.totalBytes > CUSTOM_BUILD_CONTEXT_WARN_BYTES) { + const sizeMb = (buildContextStats.totalBytes / 1_000_000).toFixed(1); + console.warn( + ` WARN: build context contains about ${sizeMb} MB across ${buildContextStats.fileCount} files.`, + ); + console.warn( + " The --from flag sends the Dockerfile's parent directory to Docker; use a dedicated directory if this is not intentional.", + ); + } buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-")); stagedDockerfile = path.join(buildCtx, "Dockerfile"); + const cleanupCustomBuildCtx = (): void => { + try { + fs.rmSync(buildCtx, { recursive: true, force: true }); + } catch { + // Best effort cleanup; the original error is more useful to the caller. + } + }; // Copy the entire parent directory as build context. try { - fs.cpSync(path.dirname(fromResolved), buildCtx, { + fs.cpSync(buildContextDir, buildCtx, { recursive: true, - filter: (src: string) => { - const base = path.basename(src); - return !["node_modules", ".git", ".venv", "__pycache__"].includes(base); - }, + filter: shouldIncludeCustomBuildContextPath, }); + // If the caller pointed at a file not named "Dockerfile", copy it to the + // location openshell expects (buildCtx/Dockerfile). + if (path.basename(fromResolved) !== "Dockerfile") { + fs.copyFileSync(fromResolved, stagedDockerfile); + } } catch (err) { + cleanupCustomBuildCtx(); const errorObject = typeof err === "object" && err !== null ? err : null; if (isErrnoException(errorObject) && errorObject.code === "EACCES") { console.error( - ` Permission denied while copying build context from: ${path.dirname(fromResolved)}`, + ` Permission denied while copying build context from: ${buildContextDir}`, ); console.error( " The --from flag uses the Dockerfile's parent directory as the Docker build context.", @@ -3746,12 +3833,6 @@ async function createSandbox( } throw err; } - // If the caller pointed at a file not named "Dockerfile", copy it to the - // location openshell expects (buildCtx/Dockerfile). - if (path.basename(fromResolved) !== "Dockerfile") { - fs.copyFileSync(fromResolved, stagedDockerfile); - } - console.log(` Using custom Dockerfile: ${fromResolved}`); } else if (agent) { const agentBuild = agentOnboard.createAgentSandbox(agent); buildCtx = agentBuild.buildCtx; @@ -4085,7 +4166,14 @@ async function createSandbox( const openshellBin = getOpenshellBinary(); for (let i = 0; i < 15; i++) { const readyMatch = runCaptureOpenshell( - ["sandbox", "exec", sandboxName, "curl", "-sf", `http://localhost:${effectiveDashboardPort}/`], + [ + "sandbox", + "exec", + sandboxName, + "curl", + "-sf", + `http://localhost:${effectiveDashboardPort}/`, + ], { ignoreError: true }, ); if (readyMatch) { @@ -5445,7 +5533,9 @@ async function setupMessagingChannels(): Promise { console.log(` ${ch.help}`); const token = normalizeCredentialValue(await prompt(` ${ch.label}: `, { secret: true })); if (token && ch.tokenFormat && !ch.tokenFormat.test(token)) { - console.log(` ✗ Invalid format. ${ch.tokenFormatHint || "Check the token and try again."}`); + console.log( + ` ✗ Invalid format. ${ch.tokenFormatHint || "Check the token and try again."}`, + ); console.log(` Skipped ${ch.name} (invalid token format)`); enabled.delete(ch.name); continue; @@ -6320,7 +6410,9 @@ async function setupPoliciesWithSelection( // the sandbox with no presets. Warn, optionally suggest the intended // variable, and fall through to the tier-derived suggestions list. console.warn(` Unsupported NEMOCLAW_POLICY_MODE: ${policyMode}`); - console.warn(" Valid values: suggested, custom, skip (aliases: default/auto, list, none/no)."); + console.warn( + " Valid values: suggested, custom, skip (aliases: default/auto, list, none/no).", + ); if (tiers.getTier(policyMode)) { console.warn( ` '${policyMode}' is a policy tier — did you mean NEMOCLAW_POLICY_TIER=${policyMode}?`, @@ -6495,10 +6587,9 @@ function getOccupiedPorts(forwardListOutput: string | null): Map */ function isPortBoundOnHost(port: number): boolean { try { - const out = runCapture( - ["lsof", "-i", `:${port}`, "-sTCP:LISTEN", "-P", "-n"], - { ignoreError: true }, - ); + const out = runCapture(["lsof", "-i", `:${port}`, "-sTCP:LISTEN", "-P", "-n"], { + ignoreError: true, + }); return !!out && out.trim().length > 0; } catch { return false; @@ -6513,7 +6604,11 @@ function isPortBoundOnHost(port: number): boolean { * non-OpenShell processes are skipped. * Throws if the entire range is exhausted. */ -function findAvailableDashboardPort(sandboxName: string, preferredPort: number, forwardListOutput: string | null): number { +function findAvailableDashboardPort( + sandboxName: string, + preferredPort: number, + forwardListOutput: string | null, +): number { const occupied = getOccupiedPorts(forwardListOutput); const preferredStr = String(preferredPort); const owner = occupied.get(preferredStr) ?? null; @@ -6531,7 +6626,9 @@ function findAvailableDashboardPort(sandboxName: string, preferredPort: number, } const owners = [...occupied.entries()] - .filter(([p]) => Number(p) >= DASHBOARD_PORT_RANGE_START && Number(p) <= DASHBOARD_PORT_RANGE_END) + .filter( + ([p]) => Number(p) >= DASHBOARD_PORT_RANGE_START && Number(p) <= DASHBOARD_PORT_RANGE_END, + ) .map(([p, s]) => ` ${p} → ${s}`) .join("\n"); throw new Error( @@ -6771,10 +6868,12 @@ function getWslHostAddress( } const runCaptureFn = options.runCapture || runCapture; const output = runCaptureFn(["hostname", "-I"], { ignoreError: true }); - return String(output || "") - .trim() - .split(/\s+/) - .filter(Boolean)[0] || null; + return ( + String(output || "") + .trim() + .split(/\s+/) + .filter(Boolean)[0] || null + ); } function getDashboardAccessInfo( @@ -6860,9 +6959,10 @@ function printDashboard( const chain = buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: wslAddr }); // Build access info inline — uses chain instead of re-deriving from env - const dashboardAccess = buildControlUiUrls(token, chain.port, chain.accessUrl).map( - (url, i) => ({ label: i === 0 ? "Dashboard" : `Alt ${i}`, url }), - ); + const dashboardAccess = buildControlUiUrls(token, chain.port, chain.accessUrl).map((url, i) => ({ + label: i === 0 ? "Dashboard" : `Alt ${i}`, + url, + })); if (wslAddr) { const wslUrl = `http://${wslAddr}:${chain.port}/${token ? `#token=${encodeURIComponent(token)}` : ""}`; const existing = dashboardAccess.find((a) => a.url === wslUrl); @@ -6870,7 +6970,10 @@ function printDashboard( else dashboardAccess.push({ label: "VS Code/WSL", url: wslUrl }); } const guidanceLines = [`Port ${chain.port} must be forwarded before opening these URLs.`]; - if (isWsl()) guidanceLines.push("WSL detected: if localhost fails in Windows, use the WSL host IP shown by `hostname -I`."); + if (isWsl()) + guidanceLines.push( + "WSL detected: if localhost fails in Windows, use the WSL host IP shown by `hostname -I`.", + ); if (dashboardAccess.length === 0) guidanceLines.push("No dashboard URLs were generated."); console.log(""); diff --git a/src/lib/sandbox-build-context.ts b/src/lib/sandbox-build-context.ts index c4ab8df593..f3ad10c223 100644 --- a/src/lib/sandbox-build-context.ts +++ b/src/lib/sandbox-build-context.ts @@ -15,6 +15,8 @@ export interface BuildContextStats { totalBytes: number; } +type BuildContextStatsFilter = (entryPath: string) => boolean; + function createBuildContextDir(tmpDir: string = os.tmpdir()): string { return fs.mkdtempSync(path.join(tmpDir, "nemoclaw-build-")); } @@ -98,13 +100,19 @@ function stageOptimizedSandboxBuildContext( return { buildCtx, stagedDockerfile }; } -function collectBuildContextStats(dir: string): BuildContextStats { +function collectBuildContextStats( + dir: string, + shouldInclude: BuildContextStatsFilter = () => true, +): BuildContextStats { let fileCount = 0; let totalBytes = 0; function walk(currentDir: string): void { for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { const entryPath = path.join(currentDir, entry.name); + if (!shouldInclude(entryPath)) { + continue; + } if (entry.isDirectory()) { walk(entryPath); continue; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index c4b82e4838..7ea90abbf3 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -5762,6 +5762,19 @@ const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); ].join("\n"), ); fs.writeFileSync(path.join(customBuildDir, "extra.txt"), "extra build context file"); + fs.writeFileSync(path.join(customBuildDir, "large.bin"), "small file with large mocked stat"); + fs.mkdirSync(path.join(customBuildDir, "node_modules", "pkg"), { recursive: true }); + fs.writeFileSync(path.join(customBuildDir, "node_modules", "pkg", "ignored.txt"), "skip me"); + fs.mkdirSync(path.join(customBuildDir, ".ssh"), { recursive: true }); + fs.writeFileSync(path.join(customBuildDir, ".ssh", "id_ed25519"), "fake test key"); + fs.mkdirSync(path.join(customBuildDir, ".aws"), { recursive: true }); + fs.writeFileSync(path.join(customBuildDir, ".aws", "credentials"), "fake test credentials"); + fs.mkdirSync(path.join(customBuildDir, "secrets"), { recursive: true }); + fs.writeFileSync(path.join(customBuildDir, "secrets", "token.txt"), "fake test token"); + fs.writeFileSync(path.join(customBuildDir, ".env.local"), "EXAMPLE=fake"); + fs.writeFileSync(path.join(customBuildDir, ".npmrc"), "registry=https://registry.example.test\n"); + fs.writeFileSync(path.join(customBuildDir, "model.pem"), "fake test certificate"); + fs.writeFileSync(path.join(customBuildDir, "credentials.json"), "{}"); fs.mkdirSync(fakeBin, { recursive: true }); fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { @@ -5779,9 +5792,20 @@ const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const fs = require("node:fs"); +const path = require("node:path"); const commands = []; let hasExtraFileAtSpawn = false; +let stagedIgnoredFilesAtSpawn = null; +const largeFilePath = ${JSON.stringify(path.join(customBuildDir, "large.bin"))}; +const originalStatSync = fs.statSync; +fs.statSync = (target, ...rest) => { + const stats = originalStatSync(target, ...rest); + if (target === largeFilePath) { + return { ...stats, size: 101_000_000 }; + } + return stats; +}; runner.run = (command, opts = {}) => { commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; @@ -5808,8 +5832,18 @@ childProcess.spawn = (...args) => { // flight — onboard deletes it once streamSandboxCreate resolves. const fromMatch = cmd.match(/--from\s+(\S+)/); if (fromMatch) { - const stagedDir = require("node:path").dirname(fromMatch[1]); - hasExtraFileAtSpawn = fs.existsSync(require("node:path").join(stagedDir, "extra.txt")); + const stagedDir = path.dirname(fromMatch[1]); + hasExtraFileAtSpawn = fs.existsSync(path.join(stagedDir, "extra.txt")); + stagedIgnoredFilesAtSpawn = { + nodeModules: fs.existsSync(path.join(stagedDir, "node_modules")), + ssh: fs.existsSync(path.join(stagedDir, ".ssh")), + aws: fs.existsSync(path.join(stagedDir, ".aws")), + secrets: fs.existsSync(path.join(stagedDir, "secrets")), + env: fs.existsSync(path.join(stagedDir, ".env.local")), + npmrc: fs.existsSync(path.join(stagedDir, ".npmrc")), + pem: fs.existsSync(path.join(stagedDir, "model.pem")), + credentialsJson: fs.existsSync(path.join(stagedDir, "credentials.json")), + }; } process.nextTick(() => { child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); @@ -5823,7 +5857,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${customDockerfilePath}); - console.log(JSON.stringify({ sandboxName, hasExtraFile: hasExtraFileAtSpawn })); + console.log(JSON.stringify({ sandboxName, hasExtraFile: hasExtraFileAtSpawn, stagedIgnoredFiles: stagedIgnoredFilesAtSpawn })); })().catch((error) => { console.error(error); process.exit(1); @@ -5852,11 +5886,25 @@ const { createSandbox } = require(${onboardPath}); assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); const payload = JSON.parse(payloadLine); assert.equal(payload.sandboxName, "my-assistant"); + assert.match(result.stdout, /Using custom Dockerfile:/); + assert.match(result.stdout, /Docker build context:/); + assert.match(result.stdout, /Docker build context:.*custom-image/); + assert.match(result.stderr, /WARN: build context contains about 101\.0 MB/); assert.equal( payload.hasExtraFile, true, "extra.txt from custom build context should be staged", ); + assert.deepEqual(payload.stagedIgnoredFiles, { + nodeModules: false, + ssh: false, + aws: false, + secrets: false, + env: false, + npmrc: false, + pem: false, + credentialsJson: false, + }); }); it("exits with an error when the --from Dockerfile path does not exist", async () => { @@ -5918,6 +5966,216 @@ const { createSandbox } = require(${onboardPath}); assert.match(result.stderr, /Custom Dockerfile not found/); }); + it("exits with an error when the --from Dockerfile path is a directory", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-dir-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-dir.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + const preflightPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "preflight.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const directoryPath = JSON.stringify(tmpDir); + + const script = String.raw` +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; +registry.registerSandbox = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${directoryPath}); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 1, "should exit 1 when fromDockerfile path is a directory"); + assert.match(result.stderr, /Custom Dockerfile path is not a file/); + }); + + it("exits clearly when the --from Dockerfile is inside an ignored context path", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-ignored-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-ignored.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + const preflightPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "preflight.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials.js")); + const ignoredDir = path.join(tmpDir, "node_modules", "pkg"); + + fs.mkdirSync(ignoredDir, { recursive: true }); + fs.writeFileSync(path.join(ignoredDir, "Dockerfile"), "FROM ubuntu:22.04\n"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const customDockerfilePath = JSON.stringify(path.join(ignoredDir, "Dockerfile")); + + const script = String.raw` +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; +registry.registerSandbox = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${customDockerfilePath}); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 1, "should exit 1 when fromDockerfile is ignored"); + assert.match(result.stderr, /inside an ignored build-context path/); + }); + + it("cleans up the custom build context when staging fails", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-cleanup-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-cleanup.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + const preflightPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "preflight.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials.js")); + const customBuildDir = path.join(tmpDir, "custom-image"); + + fs.mkdirSync(customBuildDir, { recursive: true }); + fs.writeFileSync(path.join(customBuildDir, "Dockerfile"), "FROM ubuntu:22.04\n"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const customDockerfilePath = JSON.stringify(path.join(customBuildDir, "Dockerfile")); + const customBuildDirLiteral = JSON.stringify(customBuildDir); + + const script = String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); + +let createdBuildContext = null; +const originalMkdtempSync = fs.mkdtempSync; +fs.mkdtempSync = (prefix, ...rest) => { + const dir = originalMkdtempSync(prefix, ...rest); + if (String(prefix).includes("nemoclaw-build-")) { + createdBuildContext = dir; + } + return dir; +}; +const originalCpSync = fs.cpSync; +fs.cpSync = (src, dest, options) => { + if (src === ${customBuildDirLiteral}) { + fs.writeFileSync(path.join(dest, "partial.txt"), "partial custom context"); + throw new Error("simulated custom context copy failure"); + } + return originalCpSync(src, dest, options); +}; + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; +registry.registerSandbox = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + try { + await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${customDockerfilePath}); + } catch (error) { + console.log(JSON.stringify({ + removed: Boolean(createdBuildContext) && !fs.existsSync(createdBuildContext), + message: error.message, + })); + return; + } + console.error("expected createSandbox to throw"); + process.exit(1); +})(); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout.trim().split("\n").pop()!); + assert.equal(payload.removed, true, result.stdout); + assert.match(payload.message, /simulated custom context copy failure/); + }); + it("re-prompts on invalid sandbox names instead of exiting in interactive mode", () => { const source = fs.readFileSync( path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"),