diff --git a/README.md b/README.md index 54f658f01..59fc184b9 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,17 @@ br create --title "Fix auth bug" --type task --priority 1 br close --reason "Fixed in commit abc123" ``` +## Long-running Background Shells + +proto captures the output of shell commands run with `is_background: true` to disk so detached processes never silently lose their stdout. The agent gets a stable task ID and an absolute output file path it can read at any time, plus an automatic `` on the next turn when the task exits. + +- **Output file:** `//tasks/.output` — written by the OS via shell-level redirection, so it keeps growing even after the parent wrapper exits. +- **Completion:** when the bg process exits, the next user prompt is prefixed with a `` block carrying `task_id`, `output_file`, `status` (completed/failed/killed), and `exit_code`. The agent can then `read_file` the output for results. +- **`/bg`** lists running and recently-completed background tasks. +- **`bg_stop` tool** — sends SIGTERM to the process group, escalating to SIGKILL after a 3s grace. + +This is what fixes the "agent runs an eval, can't find the results" failure mode that plagued earlier versions where backgrounded `&` commands streamed into nowhere. + ## Memory proto has a persistent memory system inspired by Claude Code. Memories are individual markdown files with YAML frontmatter, organized by type and stored per-project or globally. diff --git a/docs/architecture/divergence-from-upstream.md b/docs/architecture/divergence-from-upstream.md index a7b222f5d..615508af9 100644 --- a/docs/architecture/divergence-from-upstream.md +++ b/docs/architecture/divergence-from-upstream.md @@ -263,6 +263,18 @@ Consumed by: `task-output.ts`, `task-ready.ts`, `task-stop.ts`, `task-update.ts` (backed by `services/task-store.ts`). Distinct from the AskUserQuestion / TodoWrite split — these track long-running async work. +- **Background-shell disk capture + `bg_stop`** — `backgroundShells/` + module + `tools/bg-stop.ts`. When a shell command runs with + `is_background: true`, stdout/stderr is redirected at the shell level + to `//tasks/.output`. The OS keeps + writing even after the wrapper exits, so detached processes (long + evals, build watchers, dev servers) never silently lose output. + `BackgroundShellRegistry` tracks each task; a watcher polls the `.exit` + sentinel and marks it complete. The next user turn carries a + `` block (`task_id`, `output_file`, `status`, + `exit_code`). `bg_stop` SIGTERMs the process group with SIGKILL + fallback. Listed via `/bg`. Mirrors cc-2.18's task framework, scoped + to local shells only. **Non-Windows.** - **LSP** — `tools/lsp.ts` exposing language-server intelligence; a fork-only setting (`general.lsp`) gates it. diff --git a/docs/reference/tools/_meta.ts b/docs/reference/tools/_meta.ts index 3338d5e44..ebdfb9381 100644 --- a/docs/reference/tools/_meta.ts +++ b/docs/reference/tools/_meta.ts @@ -3,6 +3,7 @@ export default { 'file-system': 'File System', 'multi-file': 'Multi-File Read', shell: 'Shell', + 'bg-stop': 'Stop Background Shell', 'todo-write': 'Todo Write', task: 'Task', 'exit-plan-mode': 'Exit Plan Mode', diff --git a/docs/reference/tools/bg-stop.md b/docs/reference/tools/bg-stop.md new file mode 100644 index 000000000..b77080f7e --- /dev/null +++ b/docs/reference/tools/bg-stop.md @@ -0,0 +1,53 @@ +# Stop Background Shell (`bg_stop`) + +Stops a long-running background shell task spawned by [`run_shell_command`](./shell) with `is_background: true`. Sends SIGTERM to the process group, escalating to SIGKILL after a 3-second grace. + +## Parameters + +| Parameter | Required | Description | +| --------- | -------- | ---------------------------------------------------------------------- | +| `task_id` | Yes | The opaque ID returned by the original shell tool result | +| `reason` | No | Free-form reason; recorded for audit and surfaced in the result string | + +## Behavior + +1. Looks up the task in the in-session registry. +2. If the task already finished, returns its current status without signaling — no-op. +3. Sends `SIGTERM` to the process group (negative pid). Falls back to signaling the leader directly on `EPERM`. +4. Schedules a `SIGKILL` to the process group after **3 seconds** if the leader is still alive. +5. Optimistically marks the task `killed` in the registry. The watcher sees the same exit and is a no-op once the status flipped. +6. The next user prompt is prefixed with a `` block carrying `status: killed`. + +## Output + +``` +Background task "7f9c…" stopped (SIGTERM: ). +``` + +If the task was not found: + +``` +No background task with ID "". +``` + +## Finding `task_id` + +The shell tool's return value for a backgrounded command lists the task ID and output path: + +``` +Background command started. +Task ID: 7f9c… +Output file: /tmp/proto///tasks/7f9c…output +PID: 54322 +``` + +You can also list current background tasks from the TUI with the `/bg` slash command. + +## Confirmation + +`bg_stop` is classified as a `think`-kind tool — no user confirmation is required for the tool call itself. The signal still goes to the process group with the user's permissions. + +## See Also + +- [`run_shell_command`](./shell) — the `is_background: true` parameter +- `/bg` slash command — list current background tasks diff --git a/docs/reference/tools/shell.md b/docs/reference/tools/shell.md index 172df51ef..05c718f70 100644 --- a/docs/reference/tools/shell.md +++ b/docs/reference/tools/shell.md @@ -30,10 +30,29 @@ Executes a shell command. On macOS/Linux, runs with `bash -c`. On Windows, runs - Build watchers (`webpack --watch`) - Database servers (`mongod`, `redis-server`) - Any process that runs indefinitely +- Long batch jobs whose stdout you want to inspect later (evals, data processing) ## Output -Returns `Command`, `Directory`, `Stdout`, `Stderr`, `Error`, `Exit Code`, `Signal`, and `Background PIDs`. +For **foreground** commands, the tool returns `Command`, `Directory`, `Stdout`, `Stderr`, `Error`, `Exit Code`, `Signal`, and `Process Group PGID`. + +For **background** commands on non-Windows, the tool returns: + +- `Task ID` — opaque stable ID used to refer to the task later +- `Output file` — absolute path where stdout+stderr is captured (writes continue after the wrapper exits) +- `PID` — actual subprocess PID + +Read the output file at any time with the [`read_file`](./read-file) tool to inspect progress or final results — there is no need to poll. When the process exits, the **next user prompt** is prefixed with a `` block carrying `task_id`, `output_file`, `status` (`completed`/`failed`/`killed`), `exit_code`, and a human summary. + +## Background tasks + +Output files live at `//tasks/.output` and are written via shell-level redirection (`( cmd ) > file 2>&1 &`). The OS keeps writing even after the parent shell exits, so detached processes never silently lose output. + +To stop a runaway background task, use the [`bg_stop`](./bg-stop) tool with the `task_id`. It SIGTERMs the process group and escalates to SIGKILL after a 3-second grace. + +To list running and recently-completed background tasks from the TUI, run `/bg`. + +> **Windows.** Background tasks rely on POSIX shell redirection and process-group signaling. On Windows, `is_background: true` falls back to the simpler "fire and return" path used in earlier versions; output is not captured to disk. ## Confirmation diff --git a/package-lock.json b/package-lock.json index 2076d52e2..8deb07381 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@protolabsai/proto", - "version": "0.26.17", + "version": "0.26.21", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@protolabsai/proto", - "version": "0.26.17", + "version": "0.26.21", "workspaces": [ "packages/*" ], @@ -16907,7 +16907,7 @@ }, "packages/cli": { "name": "@protolabs/proto", - "version": "0.26.16", + "version": "0.26.20", "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", "@google/genai": "1.30.0", @@ -17261,7 +17261,7 @@ }, "packages/core": { "name": "@qwen-code/qwen-code-core", - "version": "0.26.16", + "version": "0.26.20", "hasInstallScript": true, "dependencies": { "@anthropic-ai/sdk": "^0.36.1", @@ -20089,7 +20089,7 @@ }, "packages/test-utils": { "name": "@qwen-code/qwen-code-test-utils", - "version": "0.26.16", + "version": "0.26.20", "dev": true, "license": "Apache-2.0", "devDependencies": { @@ -20144,7 +20144,7 @@ }, "packages/web-templates": { "name": "@qwen-code/web-templates", - "version": "0.26.16", + "version": "0.26.20", "devDependencies": { "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", @@ -20672,7 +20672,7 @@ }, "packages/webui": { "name": "@qwen-code/webui", - "version": "0.26.16", + "version": "0.26.20", "license": "MIT", "dependencies": { "markdown-it": "^14.1.0" diff --git a/package.json b/package.json index 7a19051a0..4c4f7c4e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@protolabsai/proto", - "version": "0.26.17", + "version": "0.26.21", "publishConfig": { "access": "public" }, @@ -20,7 +20,7 @@ "url": "https://github.com/protoLabsAI/protoCLI/issues" }, "config": { - "sandboxImageUri": "ghcr.io/qwenlm/qwen-code:0.26.17" + "sandboxImageUri": "ghcr.io/qwenlm/qwen-code:0.26.21" }, "scripts": { "start": "cross-env node scripts/start.js", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7348f194d..322608595 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@protolabs/proto", - "version": "0.26.16", + "version": "0.26.20", "description": "proto", "repository": { "type": "git", @@ -37,7 +37,7 @@ "dist" ], "config": { - "sandboxImageUri": "ghcr.io/qwenlm/qwen-code:0.26.17" + "sandboxImageUri": "ghcr.io/qwenlm/qwen-code:0.26.21" }, "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", diff --git a/packages/cli/src/services/BuiltinCommandLoader.ts b/packages/cli/src/services/BuiltinCommandLoader.ts index afa4ff621..45ccf706f 100644 --- a/packages/cli/src/services/BuiltinCommandLoader.ts +++ b/packages/cli/src/services/BuiltinCommandLoader.ts @@ -52,6 +52,7 @@ import { insightCommand } from '../ui/commands/insightCommand.js'; import { teamCommand } from '../ui/commands/teamCommand.js'; import { voiceCommand } from '../ui/commands/voiceCommand.js'; import { recapCommand } from '../ui/commands/recapCommand.js'; +import { bgCommand } from '../ui/commands/bgCommand.js'; /** * Loads the core, hard-coded slash commands that are an integral part @@ -114,6 +115,7 @@ export class BuiltinCommandLoader implements ICommandLoader { insightCommand, voiceCommand, recapCommand, + bgCommand, ]; return allDefinitions.filter((cmd): cmd is SlashCommand => cmd !== null); diff --git a/packages/cli/src/ui/commands/bgCommand.ts b/packages/cli/src/ui/commands/bgCommand.ts new file mode 100644 index 000000000..ad66be103 --- /dev/null +++ b/packages/cli/src/ui/commands/bgCommand.ts @@ -0,0 +1,84 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { + CommandContext, + SlashCommand, + SlashCommandActionReturn, +} from './types.js'; +import { CommandKind } from './types.js'; + +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + const s = Math.floor(ms / 1000); + if (s < 60) return `${s}s`; + const m = Math.floor(s / 60); + const rs = s % 60; + if (m < 60) return `${m}m${rs > 0 ? ` ${rs}s` : ''}`; + const h = Math.floor(m / 60); + const rm = m % 60; + return `${h}h${rm > 0 ? ` ${rm}m` : ''}`; +} + +function listAction(context: CommandContext): SlashCommandActionReturn { + const config = context.services.config; + if (!config) { + return { + type: 'message', + messageType: 'error', + content: 'Config not available.', + }; + } + const registry = config.getBackgroundShellRegistry(); + const tasks = registry.list(); + + if (tasks.length === 0) { + return { + type: 'message', + messageType: 'info', + content: + 'No background tasks. Run a shell command with is_background: true to start one.', + }; + } + + const now = Date.now(); + const lines: string[] = ['Background shell tasks (newest first):', '']; + for (const t of tasks) { + const ageMs = (t.endTime ?? now) - t.startTime; + const status = + t.status === 'running' + ? `running (${formatDuration(ageMs)})` + : `${t.status}${t.exitCode != null ? ` exit=${t.exitCode}` : ''} (ran ${formatDuration(ageMs)})`; + lines.push(` ${t.id} ${status}`); + lines.push(` cmd: ${t.command}`); + lines.push(` out: ${t.outputPath}`); + if (t.pid != null) lines.push(` pid: ${t.pid}`); + lines.push(''); + } + lines.push( + 'Stop a running task with the bg_stop tool (or kill manually).', + ); + return { + type: 'message', + messageType: 'info', + content: lines.join('\n'), + }; +} + +export const bgCommand: SlashCommand = { + name: 'bg', + description: 'List long-running background shell tasks', + kind: CommandKind.BUILT_IN, + subCommands: [ + { + name: 'list', + description: 'List background tasks', + kind: CommandKind.BUILT_IN, + action: listAction, + }, + ], + action: listAction, +}; diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 2b7f46b0b..05620140a 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -38,6 +38,7 @@ import { ToolConfirmationOutcome, logApiCancel, ApiCancelEvent, + endTurnSpan, isSupportedImageMimeType, getUnsupportedImageFormatWarning, hasFileEdits, @@ -262,13 +263,27 @@ export const useGeminiStream = ( [], ); - const [toolCalls, scheduleToolCalls, markToolsAsSubmitted] = - useReactToolScheduler( - stableOnComplete, - config, - getPreferredEditor, - onEditorClose, - ); + const [ + toolCalls, + scheduleToolCalls, + markToolsAsSubmitted, + forceCancelStaleToolCalls, + ] = useReactToolScheduler( + stableOnComplete, + config, + getPreferredEditor, + onEditorClose, + ); + + // Stable refs so cancelOngoingRequest can read the current toolCalls + // and call forceCancelStaleToolCalls without itself depending on those + // values (which would invalidate the callback every render). + const toolCallsRef = useRef(toolCalls); + toolCallsRef.current = toolCalls; + const markToolsAsSubmittedRef = useRef(markToolsAsSubmitted); + markToolsAsSubmittedRef.current = markToolsAsSubmitted; + const forceCancelStaleToolCallsRef = useRef(forceCancelStaleToolCalls); + forceCancelStaleToolCallsRef.current = forceCancelStaleToolCalls; const pendingToolCallGroupDisplay = useMemo( () => @@ -510,6 +525,40 @@ export const useGeminiStream = ( onCancelSubmit(); setIsResponding(false); setShellInputFocused(false); + + // Close any leaked OTel turn span so the recap / prompt-suggestion + // LLM calls that fire on streamingState=Idle don't get nested under + // a still-open turn span (Langfuse otherwise reports the turn as + // running until the next prompt opens a new span). + endTurnSpan('ok'); + + // Immediately flip responseSubmittedToGemini=true on every current + // toolCall. Handles the common case where a tool finished but the + // submitted flag wasn't flipped (the in-code comment at the top of + // this hook documents the same class of bug for a different cause). + const currentCallIds = toolCallsRef.current.map((tc) => tc.request.callId); + if (currentCallIds.length > 0) { + markToolsAsSubmittedRef.current(currentCallIds); + } + + // Schedule a grace-window force-clear for any tool that ignored the + // abort signal. Without this, a runaway subagent (or any tool that + // doesn't honor signal.aborted) leaves the toolCall in a non-terminal + // state, which keeps streamingState=Responding and silently drops + // every subsequent user submission via submitQuery's guard at line + // 1305. Three seconds is generous — well-behaved tools clean up in ms. + setTimeout(() => { + const cleared = forceCancelStaleToolCallsRef.current(); + if (cleared > 0) { + addItem( + { + type: MessageType.WARNING, + text: `Force-cleared ${cleared} stuck tool call(s) after cancel grace window. The underlying process(es) may still be running in the background.`, + }, + Date.now(), + ); + } + }, 3000); }, [ streamingState, addItem, @@ -1310,6 +1359,31 @@ export const useGeminiStream = ( ) { // Release the guard — we're not actually going to run if (generation !== null) queryGuardRef.current.end(generation); + // Surface the dropped submission so the user knows their input was + // not sent. Previously this was a silent `return`, which made + // protoCLI feel hung when a tool refused to honor cancel — the + // user types and gets nothing back forever. With the cancel + // handler's grace-window force-clear (~3s), this state is now + // self-resolving; the message just tells them to wait or retry. + const stateLabel = + streamingState === StreamingState.Responding + ? 'still responding' + : streamingState === StreamingState.WaitingForConfirmation + ? 'awaiting tool confirmation' + : 'backgrounded'; + addItem( + { + type: MessageType.WARNING, + text: `Input not sent — previous turn is ${stateLabel}. ${ + streamingState === StreamingState.WaitingForConfirmation + ? 'Approve or reject the pending tool call first.' + : streamingState === StreamingState.Backgrounded + ? 'Wait for the backgrounded turn to finish or press Esc to cancel.' + : 'Press Esc to cancel and try again (stuck tools auto-clear after 3s).' + }`, + }, + Date.now(), + ); return; } diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.ts b/packages/cli/src/ui/hooks/useReactToolScheduler.ts index 966c6adff..55148567f 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.ts +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.ts @@ -24,7 +24,7 @@ import { CoreToolScheduler, createDebugLogger, } from '@qwen-code/qwen-code-core'; -import { useCallback, useState, useMemo } from 'react'; +import { useCallback, useState, useMemo, useRef } from 'react'; import type { HistoryItemToolGroup, IndividualToolCallDisplay, @@ -38,6 +38,16 @@ export type ScheduleFn = ( signal: AbortSignal, ) => void; export type MarkToolsAsSubmittedFn = (callIds: string[]) => void; +/** + * Forcibly transition any non-terminal tool calls to a synthetic + * `cancelled` state with `responseSubmittedToGemini=true`. Used by the + * cancel handler to unblock `streamingState` when a tool ignores its + * AbortSignal — the underlying process may keep running, but the UI + * stops being held hostage by it. Returns the number of calls that + * were force-cancelled (so callers can decide whether to surface a + * notice). + */ +export type ForceCancelStaleToolCallsFn = () => number; export type TrackedScheduledToolCall = ScheduledToolCall & { responseSubmittedToGemini?: boolean; @@ -72,7 +82,12 @@ export function useReactToolScheduler( config: Config, getPreferredEditor: () => EditorType | undefined, onEditorClose: () => void, -): [TrackedToolCall[], ScheduleFn, MarkToolsAsSubmittedFn] { +): [ + TrackedToolCall[], + ScheduleFn, + MarkToolsAsSubmittedFn, + ForceCancelStaleToolCallsFn, +] { const [toolCallsForDisplay, setToolCallsForDisplay] = useState< TrackedToolCall[] >([]); @@ -179,7 +194,76 @@ export function useReactToolScheduler( [], ); - return [toolCallsForDisplay, schedule, markToolsAsSubmitted]; + // Track how many calls were transitioned by the most recent + // forceCancelStaleToolCalls invocation. The setter callback runs inside + // setToolCallsForDisplay's reducer, so we need a side-channel ref to + // return a synchronous count to the caller. + const lastForceCancelledCountRef = useRef(0); + + const forceCancelStaleToolCalls: ForceCancelStaleToolCallsFn = + useCallback(() => { + lastForceCancelledCountRef.current = 0; + setToolCallsForDisplay((prevCalls) => { + let changed = 0; + const next = prevCalls.map((tc): TrackedToolCall => { + // Already terminal: just guarantee the submitted flag so + // streamingState clears even if handleCompletedTools never ran. + if ( + tc.status === 'success' || + tc.status === 'error' || + tc.status === 'cancelled' + ) { + if (tc.responseSubmittedToGemini) return tc; + changed++; + return { ...tc, responseSubmittedToGemini: true }; + } + + // Non-terminal: synthesize a cancelled response. The underlying + // process may still be running — the UI stops waiting on it. + changed++; + const cancelled: TrackedCancelledToolCall = { + request: tc.request, + tool: 'tool' in tc ? tc.tool : undefined!, + invocation: 'invocation' in tc ? tc.invocation : undefined!, + status: 'cancelled', + response: { + callId: tc.request.callId, + error: undefined, + errorType: undefined, + responseParts: [ + { + functionResponse: { + id: tc.request.callId, + name: tc.request.name, + response: { + error: + 'User cancelled. Tool was force-cleared after the abort signal did not stop it within the grace window; the underlying process may still be running.', + }, + }, + }, + ], + resultDisplay: + 'Force-cancelled (tool did not honor AbortSignal in time).', + contentLength: undefined, + }, + durationMs: undefined, + outcome: tc.outcome, + responseSubmittedToGemini: true, + }; + return cancelled; + }); + lastForceCancelledCountRef.current = changed; + return changed > 0 ? next : prevCalls; + }); + return lastForceCancelledCountRef.current; + }, []); + + return [ + toolCallsForDisplay, + schedule, + markToolsAsSubmitted, + forceCancelStaleToolCalls, + ]; } /** diff --git a/packages/core/package.json b/packages/core/package.json index bd8b6e088..21b7f6384 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@qwen-code/qwen-code-core", - "version": "0.26.16", + "version": "0.26.20", "description": "proto core", "repository": { "type": "git", diff --git a/packages/core/src/backgroundShells/diskOutput.ts b/packages/core/src/backgroundShells/diskOutput.ts new file mode 100644 index 000000000..c04f8621f --- /dev/null +++ b/packages/core/src/backgroundShells/diskOutput.ts @@ -0,0 +1,125 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + * + * Disk capture helpers for background shell tasks. + * + * Unlike cc-2.18's DiskTaskOutput (which uses an in-process write queue + * fed by Node-side stream listeners), protoCLI redirects the child's + * stdout+stderr to disk at the shell level. The OS keeps writing even + * after the parent wrapper exits, which is the whole point — we never + * lose output to a detached process. + * + * These helpers compute paths and tail-read the file (with a cap so we + * never load multi-GB outputs into memory). + */ + +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import type { Config } from '../config/config.js'; + +const DEFAULT_MAX_READ_BYTES = 8 * 1024 * 1024; // 8 MB + +/** Directory that holds all background-task output files for this session. */ +export function getBackgroundTasksDir(config: Config): string { + const storage = config.storage; + if (!storage) { + throw new Error('Config has no storage; cannot resolve tasks dir'); + } + return path.join(storage.getProjectTempDir(), config.getSessionId(), 'tasks'); +} + +export function getBackgroundTaskOutputPath( + config: Config, + taskId: string, +): string { + return path.join(getBackgroundTasksDir(config), `${taskId}.output`); +} + +export function getBackgroundTaskExitPath( + config: Config, + taskId: string, +): string { + return path.join(getBackgroundTasksDir(config), `${taskId}.exit`); +} + +export function getBackgroundTaskPidPath( + config: Config, + taskId: string, +): string { + return path.join(getBackgroundTasksDir(config), `${taskId}.pid`); +} + +export async function ensureBackgroundTasksDir(config: Config): Promise { + await fs.mkdir(getBackgroundTasksDir(config), { recursive: true }); +} + +/** + * Read the tail of a task's output file. Returns the empty string if the + * file doesn't exist yet — callers should treat that as "no output yet." + */ +export async function readBackgroundTaskOutput( + config: Config, + taskId: string, + maxBytes: number = DEFAULT_MAX_READ_BYTES, +): Promise { + const outputPath = getBackgroundTaskOutputPath(config, taskId); + try { + const stat = await fs.stat(outputPath); + if (stat.size <= maxBytes) { + return await fs.readFile(outputPath, 'utf8'); + } + const fh = await fs.open(outputPath, 'r'); + try { + const offset = stat.size - maxBytes; + const buf = Buffer.alloc(maxBytes); + await fh.read(buf, 0, maxBytes, offset); + return ( + `[${Math.round(offset / 1024)}KB of earlier output omitted]\n` + + buf.toString('utf8') + ); + } finally { + await fh.close(); + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return ''; + throw err; + } +} + +/** Best-effort read of the exit code sentinel; returns null if absent. */ +export async function readBackgroundTaskExit( + config: Config, + taskId: string, +): Promise { + try { + const text = await fs.readFile( + getBackgroundTaskExitPath(config, taskId), + 'utf8', + ); + const n = Number.parseInt(text.trim(), 10); + return Number.isFinite(n) ? n : null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + return null; + } +} + +/** Best-effort read of the pid sentinel; returns null if absent. */ +export async function readBackgroundTaskPid( + config: Config, + taskId: string, +): Promise { + try { + const text = await fs.readFile( + getBackgroundTaskPidPath(config, taskId), + 'utf8', + ); + const n = Number.parseInt(text.trim(), 10); + return Number.isFinite(n) && n > 0 ? n : null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + return null; + } +} diff --git a/packages/core/src/backgroundShells/index.ts b/packages/core/src/backgroundShells/index.ts new file mode 100644 index 000000000..374ad1ec1 --- /dev/null +++ b/packages/core/src/backgroundShells/index.ts @@ -0,0 +1,11 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types.js'; +export * from './registry.js'; +export * from './diskOutput.js'; +export * from './notifications.js'; +export * from './watcher.js'; diff --git a/packages/core/src/backgroundShells/notifications.ts b/packages/core/src/backgroundShells/notifications.ts new file mode 100644 index 000000000..18490df26 --- /dev/null +++ b/packages/core/src/backgroundShells/notifications.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + * + * Builds the blocks that get prepended to the next + * user query so the model sees backgrounded tasks finishing. + */ + +import type { BackgroundShellTask } from './types.js'; + +function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>'); +} + +export function buildBackgroundTaskNotification( + task: BackgroundShellTask, +): string { + const exitLine = + task.exitCode !== undefined && task.exitCode !== null + ? `\n${task.exitCode}` + : ''; + const summary = (() => { + const desc = task.description || task.command; + switch (task.status) { + case 'completed': + return `Background command "${desc}" completed${ + task.exitCode != null ? ` (exit code ${task.exitCode})` : '' + }.`; + case 'failed': + return `Background command "${desc}" failed${ + task.exitCode != null ? ` with exit code ${task.exitCode}` : '' + }.`; + case 'killed': + return `Background command "${desc}" was stopped.`; + default: + return `Background command "${desc}" ended in unknown state.`; + } + })(); + + return [ + '', + `${task.id}`, + `${task.outputPath}`, + `${task.status}${exitLine}`, + `${escapeXml(summary)}`, + '', + '', + `Read ${task.outputPath} to see the full output.`, + ].join('\n'); +} diff --git a/packages/core/src/backgroundShells/registry.ts b/packages/core/src/backgroundShells/registry.ts new file mode 100644 index 000000000..1ede2551a --- /dev/null +++ b/packages/core/src/backgroundShells/registry.ts @@ -0,0 +1,126 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + * + * Background shell registry — tracks long-running background shell tasks + * spawned by the shell tool with `is_background: true`, plus any commands + * that get auto-backgrounded after exceeding the foreground budget. + * + * Shape mirrors cc-2.18's task framework, but scoped to local shells only + * (no agents/remote sessions). + * + * One registry per Config — singleton per session. Output files live at + * //tasks/.output. + */ + +import type { + BackgroundShellRegistrationInput, + BackgroundShellStatus, + BackgroundShellTask, +} from './types.js'; + +export class BackgroundShellRegistry { + private readonly tasks = new Map(); + private readonly listeners = new Set<() => void>(); + + register(input: BackgroundShellRegistrationInput): BackgroundShellTask { + const task: BackgroundShellTask = { + id: input.id, + command: input.command, + description: input.description, + cwd: input.cwd, + outputPath: input.outputPath, + pid: input.pid, + startTime: Date.now(), + status: 'running', + notified: false, + }; + this.tasks.set(task.id, task); + this.notify(); + return task; + } + + /** Mark a still-tracked task as exited and capture its final state. */ + markExit( + id: string, + status: Exclude, + exitCode: number | null, + ): BackgroundShellTask | undefined { + const task = this.tasks.get(id); + if (!task) return undefined; + if (task.status !== 'running') return task; + task.status = status; + task.exitCode = exitCode; + task.endTime = Date.now(); + this.notify(); + return task; + } + + /** Patch arbitrary fields. Used by bg_stop and the watcher. */ + update(id: string, patch: Partial): void { + const task = this.tasks.get(id); + if (!task) return; + Object.assign(task, patch); + this.notify(); + } + + get(id: string): BackgroundShellTask | undefined { + return this.tasks.get(id); + } + + /** All tasks, newest first. */ + list(): BackgroundShellTask[] { + return [...this.tasks.values()].sort((a, b) => b.startTime - a.startTime); + } + + /** Currently-running tasks. */ + running(): BackgroundShellTask[] { + return this.list().filter((t) => t.status === 'running'); + } + + /** + * Returns completed-but-unnotified tasks and atomically marks them + * notified. Called by the client just before sending a user query so + * the model sees a for each task that finished + * since the last turn. + */ + drainPendingNotifications(): BackgroundShellTask[] { + const drained: BackgroundShellTask[] = []; + for (const task of this.tasks.values()) { + if (task.status !== 'running' && !task.notified) { + task.notified = true; + drained.push({ ...task }); + } + } + if (drained.length > 0) this.notify(); + return drained; + } + + /** Subscribe to any change. Returns an unsubscribe handle. */ + subscribe(listener: () => void): () => void { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + + /** + * Drop all registry state. Used by tests and when a session ends — + * does NOT delete the on-disk output files. + */ + clear(): void { + this.tasks.clear(); + this.notify(); + } + + private notify(): void { + for (const listener of [...this.listeners]) { + try { + listener(); + } catch { + // listener errors must not bring down the registry + } + } + } +} diff --git a/packages/core/src/backgroundShells/types.ts b/packages/core/src/backgroundShells/types.ts new file mode 100644 index 000000000..015830708 --- /dev/null +++ b/packages/core/src/backgroundShells/types.ts @@ -0,0 +1,44 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + */ + +export type BackgroundShellStatus = + | 'running' + | 'completed' + | 'failed' + | 'killed'; + +export interface BackgroundShellTask { + /** Stable opaque ID — used in tool returns, /bg, and bg_stop. */ + id: string; + /** The command line as the model wrote it. */ + command: string; + /** Short human-readable description (often a truncated command). */ + description: string; + /** Working directory. */ + cwd: string; + /** ms epoch when the task was registered. */ + startTime: number; + /** ms epoch when the task exited (any terminal status). */ + endTime?: number; + status: BackgroundShellStatus; + /** Process exit code if known. Null when killed by signal with no clean code. */ + exitCode?: number | null; + /** Absolute path where stdout+stderr is being captured. */ + outputPath: string; + /** PID of the spawned shell wrapper (used by bg_stop to signal the group). */ + pid?: number; + /** True once the model has been informed of completion via task_notification. */ + notified: boolean; +} + +export interface BackgroundShellRegistrationInput { + id: string; + command: string; + description: string; + cwd: string; + outputPath: string; + pid?: number; +} diff --git a/packages/core/src/backgroundShells/watcher.ts b/packages/core/src/backgroundShells/watcher.ts new file mode 100644 index 000000000..15328a8c9 --- /dev/null +++ b/packages/core/src/backgroundShells/watcher.ts @@ -0,0 +1,109 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + * + * Background-shell watcher. + * + * Polls a registered task's `.exit` sentinel file (and falls back to a + * liveness check on the captured PID) to detect process exit. Updates + * the registry on terminal status — no streaming required, since stdout + * is already going to disk via the shell-level redirection in shell.ts. + */ + +import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; +import { readBackgroundTaskExit, readBackgroundTaskPid } from './diskOutput.js'; + +const debugLogger = createDebugLogger('BG_SHELL_WATCHER'); + +const POLL_INTERVAL_MS = 1000; +/** Hard ceiling so a stuck watcher can't run forever. ~7 days. */ +const MAX_POLL_ATTEMPTS = 60 * 60 * 24 * 7; + +/** True iff a process with this pid is still alive (best-effort, POSIX). */ +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + // EPERM = exists but we can't signal it (still alive). ESRCH = gone. + return code === 'EPERM'; + } +} + +/** + * Begin polling for a task's exit. Resolves once the task has been marked + * with a terminal status. Errors are logged and swallowed — this runs as + * a fire-and-forget side effect. + */ +export function startBackgroundShellWatcher( + config: Config, + taskId: string, +): void { + void poll(config, taskId).catch((err) => { + debugLogger.warn( + `[bg-shell-watcher] task ${taskId}: poll failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); +} + +async function poll(config: Config, taskId: string): Promise { + const registry = config.getBackgroundShellRegistry(); + let attempts = 0; + let pidCache: number | null = null; + + while (attempts < MAX_POLL_ATTEMPTS) { + attempts++; + const task = registry.get(taskId); + if (!task) return; // registry was cleared, nothing to watch + if (task.status !== 'running') return; // someone else (bg_stop) finalized it + + if (pidCache === null) { + pidCache = await readBackgroundTaskPid(config, taskId); + if (pidCache !== null && task.pid === undefined) { + registry.update(taskId, { pid: pidCache }); + } + } + + // Prefer the exit-code sentinel: it's written *after* the process + // exits and tells us the actual code, including for failures. + const exitCode = await readBackgroundTaskExit(config, taskId); + if (exitCode !== null) { + const status = exitCode === 0 ? 'completed' : 'failed'; + registry.markExit(taskId, status, exitCode); + return; + } + + // Fallback: pid disappeared but we somehow missed the sentinel + // (process killed by SIGKILL, oom, etc). Mark as failed with + // unknown exit code so the next-turn notification still fires. + if (pidCache !== null && !isPidAlive(pidCache)) { + // Give the sentinel one more poll cycle to land — common race + // where the process exited but the parent shell hasn't flushed + // `echo $? > .exit` yet. + await sleep(POLL_INTERVAL_MS); + const lateExit = await readBackgroundTaskExit(config, taskId); + if (lateExit !== null) { + const status = lateExit === 0 ? 'completed' : 'failed'; + registry.markExit(taskId, status, lateExit); + return; + } + registry.markExit(taskId, 'failed', null); + return; + } + + await sleep(POLL_INTERVAL_MS); + } + + debugLogger.warn( + `[bg-shell-watcher] task ${taskId}: gave up after ${attempts} attempts`, + ); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 7a5eaacda..119f47f87 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -67,6 +67,7 @@ import { TaskUpdateTool } from '../tools/task-update.js'; import { TaskStopTool } from '../tools/task-stop.js'; import { TaskOutputTool } from '../tools/task-output.js'; import { TaskReadyTool } from '../tools/task-ready.js'; +import { BgStopTool } from '../tools/bg-stop.js'; import { ToolRegistry } from '../tools/tool-registry.js'; import { WebFetchTool } from '../tools/web-fetch.js'; import { WebSearchTool } from '../tools/web-search/index.js'; @@ -133,6 +134,7 @@ import { import { DEFAULT_QWEN_EMBEDDING_MODEL } from './models.js'; import { Storage } from './storage.js'; import { TaskStore } from '../services/task-store.js'; +import { BackgroundShellRegistry } from '../backgroundShells/registry.js'; import { ChatRecordingService } from '../services/chatRecordingService.js'; import { MemoryConsolidationService, @@ -667,6 +669,7 @@ export class Config { ) => Promise; private initialized: boolean = false; private taskStore?: TaskStore; + private backgroundShellRegistry?: BackgroundShellRegistry; private memoryConsolidationService?: MemoryConsolidationService; private readonly memoryConsolidationConfig?: Partial; readonly storage: Storage; @@ -1219,6 +1222,18 @@ export class Config { return this.taskStore; } + /** + * Returns the registry of long-running background shell tasks. Lazy + * — only constructed once a task is registered. Disk capture lives at + * //tasks/.output. + */ + getBackgroundShellRegistry(): BackgroundShellRegistry { + if (!this.backgroundShellRegistry) { + this.backgroundShellRegistry = new BackgroundShellRegistry(); + } + return this.backgroundShellRegistry; + } + /** * Returns warnings generated during configuration resolution. * These warnings are collected from model configuration resolution @@ -2363,6 +2378,7 @@ export class Config { await registerCoreTool(TaskStopTool, this); await registerCoreTool(TaskOutputTool, this); await registerCoreTool(TaskReadyTool, this); + await registerCoreTool(BgStopTool, this); await registerCoreTool(AskUserQuestionTool, this); !this.sdkMode && (await registerCoreTool(ExitPlanModeTool, this)); await registerCoreTool(WebFetchTool, this); diff --git a/packages/core/src/core/__snapshots__/prompts.test.ts.snap b/packages/core/src/core/__snapshots__/prompts.test.ts.snap index f9e9ccb82..9e7361733 100644 --- a/packages/core/src/core/__snapshots__/prompts.test.ts.snap +++ b/packages/core/src/core/__snapshots__/prompts.test.ts.snap @@ -157,7 +157,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -422,7 +422,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -699,7 +699,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -959,7 +959,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -1219,7 +1219,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -1479,7 +1479,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -1739,7 +1739,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -1999,7 +1999,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -2259,7 +2259,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -2519,7 +2519,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -2802,7 +2802,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -3148,7 +3148,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -3431,7 +3431,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -3773,7 +3773,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. @@ -4033,7 +4033,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like 'read_file' or 'write_file'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the 'run_shell_command' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to 'run_shell_command'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with 'read_file' to inspect progress or final results, and stop a runaway task with 'bg_stop' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('task_create', 'task_update', 'task_list') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the 'agent' tool in order to reduce context usage. You should proactively use the 'agent' tool with specialized agents when the task at hand matches the agent's description. diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index b69afaff3..f2c8af40a 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -31,6 +31,7 @@ import { getPlanModeSystemReminder, getSubagentSystemReminder, } from './prompts.js'; +import { buildBackgroundTaskNotification } from '../backgroundShells/notifications.js'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import { CompressionStatus, @@ -820,6 +821,14 @@ export class GeminiClient { } } + // add background-task completion notifications. Drained on read so + // each completed task is announced exactly once. + const bgRegistry = this.config.getBackgroundShellRegistry?.(); + const bgPending = bgRegistry?.drainPendingNotifications() ?? []; + for (const task of bgPending) { + systemReminders.push(buildBackgroundTaskNotification(task)); + } + requestToSent = [...systemReminders, ...requestToSent]; } diff --git a/packages/core/src/core/prompts.ts b/packages/core/src/core/prompts.ts index e74ee4df7..dbc5c0756 100644 --- a/packages/core/src/core/prompts.ts +++ b/packages/core/src/core/prompts.ts @@ -369,7 +369,7 @@ When you encounter an obstacle, do not resort to destructive shortcuts. Investig - **File Paths:** Always use absolute paths when referring to files with tools like '${ToolNames.READ_FILE}' or '${ToolNames.WRITE_FILE}'. Relative paths are not supported. You must provide an absolute path. - **Parallelism:** Execute multiple independent tool calls in parallel when feasible (i.e. searching the codebase). - **Command Execution:** Use the '${ToolNames.SHELL}' tool for running shell commands, remembering the safety rule to explain modifying commands first. -- **Background Processes:** Use background processes (via \`&\`) for commands that are unlikely to stop on their own, e.g. \`node server.js &\`. If unsure, ask the user. +- **Background Processes:** For commands that are unlikely to stop on their own (servers, watchers) or that you'll want to inspect later (long evals, batch jobs), pass \`is_background: true\` to '${ToolNames.SHELL}'. The tool returns a \`Task ID\` and an \`Output file\` path; stdout/stderr is redirected to that file at the shell level so output is never lost. A \`\` arrives on the next turn when the task exits — do not poll. Read the output file with '${ToolNames.READ_FILE}' to inspect progress or final results, and stop a runaway task with '${ToolNames.BG_STOP}' (passing the \`task_id\`). - **Interactive Commands:** Try to avoid shell commands that are likely to require user interaction (e.g. \`git rebase -i\`). Use non-interactive versions of commands (e.g. \`npm init -y\` instead of \`npm init\`) when available, and otherwise remind the user that interactive shell commands are not supported and may cause hangs until canceled by the user. - **Task Management:** Use the task management tools ('${ToolNames.TASK_CREATE}', '${ToolNames.TASK_UPDATE}', '${ToolNames.TASK_LIST}') proactively for complex, multi-step tasks to track progress and provide visibility to users. These tools help organize work systematically and ensure no requirements are missed. - **Subagent Delegation:** When doing file search, prefer to use the '${ToolNames.AGENT}' tool in order to reduce context usage. You should proactively use the '${ToolNames.AGENT}' tool with specialized agents when the task at hand matches the agent's description. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d8c369f38..10a997d05 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -99,6 +99,7 @@ export * from './tools/task-list.js'; export * from './tools/task-update.js'; export * from './tools/task-stop.js'; export * from './tools/task-output.js'; +export * from './tools/bg-stop.js'; export * from './services/task-store.js'; export * from './tools/tool-error.js'; export * from './tools/tool-registry.js'; @@ -244,6 +245,12 @@ export * from './followup/index.js'; export * from './recap/index.js'; +// ============================================================================ +// Background shell tasks (long-running `is_background: true` commands) +// ============================================================================ + +export * from './backgroundShells/index.js'; + // ============================================================================ // Utilities // ============================================================================ diff --git a/packages/core/src/tools/__snapshots__/shell.test.ts.snap b/packages/core/src/tools/__snapshots__/shell.test.ts.snap index 9dfd54cf0..0bd076759 100644 --- a/packages/core/src/tools/__snapshots__/shell.test.ts.snap +++ b/packages/core/src/tools/__snapshots__/shell.test.ts.snap @@ -38,6 +38,7 @@ IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO N - Database servers: \`mongod\`, \`mysql\`, \`redis-server\` - Web servers: \`python -m http.server\`, \`php -S localhost:8000\` - Any command expected to run indefinitely until manually stopped + - Long-running batch jobs whose stdout you'll want to inspect later (evals, data processing, CI runs) - Command is executed as a subprocess that leads its own process group. Command process group can be terminated as \`kill -- -PGID\` or signaled as \`kill -s SIGNAL -- -PGID\`. - Use foreground execution (is_background: false) for: @@ -46,6 +47,12 @@ IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO N - Installation commands: \`npm install\`, \`pip install\` - Git operations: \`git commit\`, \`git push\` - Test runs: \`npm test\`, \`pytest\` + +**How background tasks work (non-Windows):** +- The tool result returns a stable \`Task ID\` and an absolute \`Output file\` path. stdout and stderr are redirected at the shell level, so the OS keeps writing even after this tool returns. **Output is never lost** — read the file with the read_file tool whenever you want to inspect progress or final results. +- When the process exits, the next user turn includes a \`\` block with the task's \`status\` (completed | failed | killed), \`exit_code\`, and \`output_file\` path. **Do not poll** — the notification arrives automatically. +- Stop a runaway background task with the \`bg_stop\` tool, passing the \`task_id\` from the original tool result. It SIGTERMs the process group and escalates to SIGKILL after a short grace period. +- Output files live under the project temp dir; they survive across turns within a session and can grow large. Prefer the read_file tool (which tail-reads with a cap) over a shell \`cat\` for large outputs. " `; @@ -87,6 +94,7 @@ IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO N - Database servers: \`mongod\`, \`mysql\`, \`redis-server\` - Web servers: \`python -m http.server\`, \`php -S localhost:8000\` - Any command expected to run indefinitely until manually stopped + - Long-running batch jobs whose stdout you'll want to inspect later (evals, data processing, CI runs) - Use foreground execution (is_background: false) for: - One-time commands: \`ls\`, \`cat\`, \`grep\` @@ -94,5 +102,11 @@ IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO N - Installation commands: \`npm install\`, \`pip install\` - Git operations: \`git commit\`, \`git push\` - Test runs: \`npm test\`, \`pytest\` + +**How background tasks work (non-Windows):** +- The tool result returns a stable \`Task ID\` and an absolute \`Output file\` path. stdout and stderr are redirected at the shell level, so the OS keeps writing even after this tool returns. **Output is never lost** — read the file with the read_file tool whenever you want to inspect progress or final results. +- When the process exits, the next user turn includes a \`\` block with the task's \`status\` (completed | failed | killed), \`exit_code\`, and \`output_file\` path. **Do not poll** — the notification arrives automatically. +- Stop a runaway background task with the \`bg_stop\` tool, passing the \`task_id\` from the original tool result. It SIGTERMs the process group and escalates to SIGKILL after a short grace period. +- Output files live under the project temp dir; they survive across turns within a session and can grow large. Prefer the read_file tool (which tail-reads with a cap) over a shell \`cat\` for large outputs. " `; diff --git a/packages/core/src/tools/bg-stop.ts b/packages/core/src/tools/bg-stop.ts new file mode 100644 index 000000000..8bbf6979e --- /dev/null +++ b/packages/core/src/tools/bg-stop.ts @@ -0,0 +1,168 @@ +/** + * @license + * Copyright 2025 protoLabs Studio + * SPDX-License-Identifier: Apache-2.0 + * + * bg_stop — kill a long-running background shell task spawned by the + * shell tool with `is_background: true`. Sends SIGTERM to the process + * group, then SIGKILL after a short grace period if still alive. + */ + +import type { ToolInvocation, ToolResult } from './tools.js'; +import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js'; +import { ToolNames, ToolDisplayNames } from './tool-names.js'; +import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; + +const debugLogger = createDebugLogger('BG_STOP'); + +const SIGKILL_GRACE_MS = 3000; + +export interface BgStopParams { + task_id: string; + reason?: string; +} + +function killProcessGroup(pid: number, signal: NodeJS.Signals): boolean { + try { + // Negative pid signals the whole process group, killing children too. + process.kill(-pid, signal); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ESRCH') return false; // already gone + if (code === 'EPERM') { + // Fall back to signaling just the leader. + try { + process.kill(pid, signal); + return true; + } catch { + return false; + } + } + debugLogger.warn( + `[bg_stop] kill(-${pid}, ${signal}) failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return false; + } +} + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +class BgStopToolInvocation extends BaseToolInvocation< + BgStopParams, + ToolResult +> { + constructor( + private readonly config: Config, + params: BgStopParams, + ) { + super(params); + } + + getDescription(): string { + return `Stop background task: ${this.params.task_id}${ + this.params.reason ? ` (${this.params.reason})` : '' + }`; + } + + async execute(_signal: AbortSignal): Promise { + const registry = this.config.getBackgroundShellRegistry(); + const task = registry.get(this.params.task_id); + + if (!task) { + const msg = `No background task with ID "${this.params.task_id}".`; + return { + llmContent: msg, + returnDisplay: msg, + error: { message: msg }, + }; + } + + if (task.status !== 'running') { + const msg = `Background task "${this.params.task_id}" already finished with status "${task.status}".`; + return { + llmContent: msg, + returnDisplay: msg, + }; + } + + if (!task.pid) { + // No PID captured — can't signal. Mark killed so the next-turn + // notification fires anyway. + registry.markExit(task.id, 'killed', null); + const msg = `Background task "${task.id}" had no captured PID; marked killed without signal.`; + return { llmContent: msg, returnDisplay: msg }; + } + + const sigtermOk = killProcessGroup(task.pid, 'SIGTERM'); + if (!sigtermOk && !isAlive(task.pid)) { + registry.markExit(task.id, 'killed', null); + const msg = `Background task "${task.id}" was already gone.`; + return { llmContent: msg, returnDisplay: msg }; + } + + // SIGKILL fallback after a short grace period. + setTimeout(() => { + if (task.pid && isAlive(task.pid)) { + killProcessGroup(task.pid, 'SIGKILL'); + } + }, SIGKILL_GRACE_MS).unref(); + + // Optimistically mark as killed in the registry — the watcher will + // also see the process disappear and is a no-op once status flipped. + registry.markExit(task.id, 'killed', null); + + const msg = `Background task "${task.id}" stopped (SIGTERM${ + this.params.reason ? `: ${this.params.reason}` : '' + }).`; + return { + llmContent: msg, + returnDisplay: msg, + }; + } +} + +export class BgStopTool extends BaseDeclarativeTool { + static readonly Name: string = ToolNames.BG_STOP; + + constructor(private readonly config: Config) { + super( + BgStopTool.Name, + ToolDisplayNames.BG_STOP, + 'Stops a long-running background shell task spawned by the shell tool with is_background=true. Sends SIGTERM to the process group, escalating to SIGKILL after a short grace period.', + Kind.Think, + { + type: 'object', + properties: { + task_id: { + type: 'string', + description: 'The ID of the background task to stop.', + minLength: 1, + }, + reason: { + type: 'string', + description: 'Optional reason for stopping (audit only).', + }, + }, + required: ['task_id'], + $schema: 'http://json-schema.org/draft-07/schema#', + }, + ); + } + + protected createInvocation( + params: BgStopParams, + ): ToolInvocation { + return new BgStopToolInvocation(this.config, params); + } +} diff --git a/packages/core/src/tools/shell.test.ts b/packages/core/src/tools/shell.test.ts index d7fef7fff..90be0aef8 100644 --- a/packages/core/src/tools/shell.test.ts +++ b/packages/core/src/tools/shell.test.ts @@ -75,6 +75,13 @@ describe('ShellTool', () => { email: 'qwen-coder@alibabacloud.com', }), getShouldUseNodePtyShell: vi.fn().mockReturnValue(false), + getSessionId: vi.fn().mockReturnValue('test-session'), + getBackgroundShellRegistry: vi.fn().mockReturnValue({ + register: vi.fn(), + update: vi.fn(), + markExit: vi.fn(), + get: vi.fn(), + }), } as unknown as Config; shellTool = new ShellTool(mockConfig); @@ -84,6 +91,9 @@ describe('ShellTool', () => { (vi.mocked(crypto.randomBytes) as Mock).mockReturnValue( Buffer.from('abcdef', 'hex'), ); + vi.mocked(crypto.randomUUID).mockReturnValue( + '00000000-0000-0000-0000-000000000000', + ); // Capture the output callback to simulate streaming events from the service mockShellExecutionService.mockImplementation((_cmd, _cwd, callback) => { @@ -288,7 +298,15 @@ describe('ShellTool', () => { const result = await promise; const tmpFile = path.join(os.tmpdir(), 'shell_pgrep_abcdef.tmp'); - const wrappedCommand = `{ my-command & }; __code=$?; pgrep -g 0 >${tmpFile} 2>&1; exit $__code;`; + const tasksDir = + '/tmp/qwen-temp/test-session/tasks/00000000-0000-0000-0000-000000000000'; + const wrappedCommand = [ + `{ ( my-command ) > "${tasksDir}.output" 2>&1; echo $? > "${tasksDir}.exit"; } &`, + `__bgpid=$!`, + `echo $__bgpid > "${tasksDir}.pid"`, + `pgrep -g 0 > ${tmpFile} 2>&1`, + `exit 0`, + ].join('\n'); expect(mockShellExecutionService).toHaveBeenCalledWith( wrappedCommand, '/test/dir', @@ -297,7 +315,11 @@ describe('ShellTool', () => { false, {}, ); - expect(result.llmContent).toContain('PIDs: 54322'); + expect(result.llmContent).toContain('Background command started.'); + expect(result.llmContent).toContain( + 'Task ID: 00000000-0000-0000-0000-000000000000', + ); + expect(result.llmContent).toContain(`Output file: ${tasksDir}.output`); expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalledWith(tmpFile); }); @@ -315,7 +337,15 @@ describe('ShellTool', () => { await promise; const tmpFile = path.join(os.tmpdir(), 'shell_pgrep_abcdef.tmp'); - const wrappedCommand = `{ npm start & }; __code=$?; pgrep -g 0 >${tmpFile} 2>&1; exit $__code;`; + const tasksDir = + '/tmp/qwen-temp/test-session/tasks/00000000-0000-0000-0000-000000000000'; + const wrappedCommand = [ + `{ ( npm start ) > "${tasksDir}.output" 2>&1; echo $? > "${tasksDir}.exit"; } &`, + `__bgpid=$!`, + `echo $__bgpid > "${tasksDir}.pid"`, + `pgrep -g 0 > ${tmpFile} 2>&1`, + `exit 0`, + ].join('\n'); expect(mockShellExecutionService).toHaveBeenCalledWith( wrappedCommand, expect.any(String), @@ -340,7 +370,15 @@ describe('ShellTool', () => { await promise; const tmpFile = path.join(os.tmpdir(), 'shell_pgrep_abcdef.tmp'); - const wrappedCommand = `{ npm start & }; __code=$?; pgrep -g 0 >${tmpFile} 2>&1; exit $__code;`; + const tasksDir = + '/tmp/qwen-temp/test-session/tasks/00000000-0000-0000-0000-000000000000'; + const wrappedCommand = [ + `{ ( npm start ) > "${tasksDir}.output" 2>&1; echo $? > "${tasksDir}.exit"; } &`, + `__bgpid=$!`, + `echo $__bgpid > "${tasksDir}.pid"`, + `pgrep -g 0 > ${tmpFile} 2>&1`, + `exit 0`, + ].join('\n'); expect(mockShellExecutionService).toHaveBeenCalledWith( wrappedCommand, expect.any(String), diff --git a/packages/core/src/tools/shell.ts b/packages/core/src/tools/shell.ts index 8af255538..c2a7964fd 100644 --- a/packages/core/src/tools/shell.ts +++ b/packages/core/src/tools/shell.ts @@ -43,6 +43,13 @@ import { isShellCommandReadOnlyAST, extractCommandRules, } from '../utils/shellAstParser.js'; +import { + getBackgroundTaskExitPath, + getBackgroundTaskOutputPath, + getBackgroundTaskPidPath, + readBackgroundTaskPid, +} from '../backgroundShells/diskOutput.js'; +import { startBackgroundShellWatcher } from '../backgroundShells/watcher.js'; const debugLogger = createDebugLogger('SHELL'); @@ -232,11 +239,39 @@ export class ShellToolInvocation extends BaseToolInvocation< .toString('hex')}.tmp`; const tempFilePath = path.join(os.tmpdir(), tempFileName); + // Background-task disk capture (non-Windows only). When is_background + // is true we redirect stdout/stderr at the shell level into a per-task + // file so the OS keeps writing even after the wrapper exits — the + // detached process can no longer drop output on the floor. + const shouldRunInBackground = this.params.is_background; + const useDiskCapture = shouldRunInBackground && !isWindows; + const backgroundTaskId = useDiskCapture ? crypto.randomUUID() : undefined; + let backgroundOutputPath: string | undefined; + let backgroundExitPath: string | undefined; + let backgroundPidPath: string | undefined; + if (backgroundTaskId) { + backgroundOutputPath = getBackgroundTaskOutputPath( + this.config, + backgroundTaskId, + ); + backgroundExitPath = getBackgroundTaskExitPath( + this.config, + backgroundTaskId, + ); + backgroundPidPath = getBackgroundTaskPidPath( + this.config, + backgroundTaskId, + ); + // Synchronous mkdir so the directory exists by the time the shell + // wrapper tries to redirect into it. The async ensureBackgroundTasksDir + // helper would put a microtask between us and ShellExecutionService. + fs.mkdirSync(path.dirname(backgroundOutputPath), { recursive: true }); + } + try { // Add co-author to git commit commands const processedCommand = this.addCoAuthorToGitCommit(strippedCommand); - const shouldRunInBackground = this.params.is_background; let finalCommand = processedCommand; // On non-Windows, use & to run in background. @@ -254,19 +289,42 @@ export class ShellToolInvocation extends BaseToolInvocation< // On Windows, we rely on the race logic below to handle background tasks. // We just ensure the command string is clean. if (isWindows && shouldRunInBackground) { - finalCommand = finalCommand.trim().replace(/&+$/, '').trim(); + // Strip any trailing & without a regex — `&+$` looked benign but + // tripped CodeQL's polynomial-redos rule. A simple loop has no + // backtracking risk and the same intent. + let s = finalCommand.trim(); + while (s.endsWith('&')) s = s.slice(0, -1); + finalCommand = s.trimEnd(); } - // On non-Windows background commands, wrap with pgrep to capture - // subprocess PIDs so we can report them to the user. - const commandToExecute = - !isWindows && shouldRunInBackground - ? (() => { - let command = finalCommand.trim(); - if (!command.endsWith('&')) command += ';'; - return `{ ${command} }; __code=$?; pgrep -g 0 >${tempFilePath} 2>&1; exit $__code;`; - })() - : finalCommand; + // Build the shell wrapper. + // - For backgrounded commands with disk capture (non-Windows): redirect + // stdout+stderr to /.output, capture the bg PID and exit + // code via sentinel files so a Node-side watcher can detect exit. + // - For backgrounded commands on Windows: pass through unchanged + // (relies on the race-return logic below; no & or pgrep wrapper). + // - For foreground commands: pass through unchanged. + const commandToExecute = (() => { + if (!shouldRunInBackground || !useDiskCapture) return finalCommand; + // Strip trailing & (and any preceding whitespace) — we'll re-add it + // on the subshell wrapper. Plain string ops instead of `\s*&\s*$` + // because the regex tripped CodeQL's polynomial-redos rule. + let inner = finalCommand.trim(); + if (inner.endsWith('&')) inner = inner.slice(0, -1).trimEnd(); + return [ + // () >output 2>&1; echo $? >exit — runs in its own subshell + // detached with `&`, so the OS keeps writing even after our + // wrapper exits. Sentinel files let the Node watcher detect + // completion without re-reading stdout. + `{ ( ${inner} ) > "${backgroundOutputPath}" 2>&1; echo $? > "${backgroundExitPath}"; } &`, + // capture the spawned subshell's PID + `__bgpid=$!`, + `echo $__bgpid > "${backgroundPidPath}"`, + // pgrep for legacy "child PIDs" reporting + `pgrep -g 0 > ${tempFilePath} 2>&1`, + `exit 0`, + ].join('\n'); + })(); const cwd = this.params.directory || this.config.getTargetDir(); @@ -373,6 +431,41 @@ export class ShellToolInvocation extends BaseToolInvocation< : ''; const killHint = ' (Use kill to stop)'; + // For non-Windows background tasks: register with the registry, + // start the watcher, and return the disk-capture file path so the + // model can Read it later. + if (backgroundTaskId && backgroundOutputPath) { + const realPid = await readBackgroundTaskPid( + this.config, + backgroundTaskId, + ); + const registry = this.config.getBackgroundShellRegistry(); + registry.register({ + id: backgroundTaskId, + command: this.params.command, + description: this.params.description ?? this.params.command, + cwd, + outputPath: backgroundOutputPath, + pid: realPid ?? undefined, + }); + startBackgroundShellWatcher(this.config, backgroundTaskId); + + const lines = [ + `Background command started.`, + `Task ID: ${backgroundTaskId}`, + `Output file: ${backgroundOutputPath}`, + `PID: ${realPid ?? '(unknown)'}${bgPidMsg ? ` Children${bgPidMsg.replace(/^ PIDs:/, ':')}` : ''}`, + `Read the output file at any time to check progress. You will`, + `be notified via when the task completes.`, + `Stop early with the bg_stop tool (task_id="${backgroundTaskId}").`, + ]; + const message = lines.join('\n'); + return { + llmContent: message, + returnDisplay: message, + }; + } + return { llmContent: `Background command started.${bgPidMsg}${killHint}`, returnDisplay: `Background command started.${bgPidMsg}${killHint}`, @@ -587,6 +680,7 @@ IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO N - Database servers: \`mongod\`, \`mysql\`, \`redis-server\` - Web servers: \`python -m http.server\`, \`php -S localhost:8000\` - Any command expected to run indefinitely until manually stopped + - Long-running batch jobs whose stdout you'll want to inspect later (evals, data processing, CI runs) ${processGroupNote} - Use foreground execution (is_background: false) for: - One-time commands: \`ls\`, \`cat\`, \`grep\` @@ -594,6 +688,12 @@ ${processGroupNote} - Installation commands: \`npm install\`, \`pip install\` - Git operations: \`git commit\`, \`git push\` - Test runs: \`npm test\`, \`pytest\` + +**How background tasks work (non-Windows):** +- The tool result returns a stable \`Task ID\` and an absolute \`Output file\` path. stdout and stderr are redirected at the shell level, so the OS keeps writing even after this tool returns. **Output is never lost** — read the file with the ${ToolNames.READ_FILE} tool whenever you want to inspect progress or final results. +- When the process exits, the next user turn includes a \`\` block with the task's \`status\` (completed | failed | killed), \`exit_code\`, and \`output_file\` path. **Do not poll** — the notification arrives automatically. +- Stop a runaway background task with the \`${ToolNames.BG_STOP}\` tool, passing the \`task_id\` from the original tool result. It SIGTERMs the process group and escalates to SIGKILL after a short grace period. +- Output files live under the project temp dir; they survive across turns within a session and can grow large. Prefer the ${ToolNames.READ_FILE} tool (which tail-reads with a cap) over a shell \`cat\` for large outputs. `; } @@ -627,7 +727,7 @@ export class ShellTool extends BaseDeclarativeTool< is_background: { type: 'boolean', description: - 'Optional: Whether to run the command in background. If not specified, defaults to false (foreground execution). Explicitly set to true for long-running processes like development servers, watchers, or daemons that should continue running without blocking further commands.', + 'Optional: Whether to run the command in background. Defaults to false. Set to true for long-running processes (servers, watchers, evals, batch jobs). Background tasks return a Task ID and an Output file path; stdout/stderr is redirected to that file by the OS so output survives even after this tool returns. You will receive a on the next turn when the task exits — do not poll. Stop a runaway task with the bg_stop tool.', }, timeout: { type: 'number', diff --git a/packages/core/src/tools/tool-names.ts b/packages/core/src/tools/tool-names.ts index 1f0326372..c455736ae 100644 --- a/packages/core/src/tools/tool-names.ts +++ b/packages/core/src/tools/tool-names.ts @@ -37,6 +37,7 @@ export const ToolNames = { CRON_DELETE: 'cron_delete', REPO_MAP: 'repo_map', BROWSER: 'browser', + BG_STOP: 'bg_stop', } as const; /** @@ -72,6 +73,7 @@ export const ToolDisplayNames = { CRON_DELETE: 'CronDelete', REPO_MAP: 'RepoMap', BROWSER: 'Browser', + BG_STOP: 'BgStop', } as const; // Migration from old tool names to new tool names diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index eb0de6a1a..ae735fba4 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,6 +1,6 @@ { "name": "@qwen-code/qwen-code-test-utils", - "version": "0.26.16", + "version": "0.26.20", "private": true, "main": "src/index.ts", "license": "Apache-2.0", diff --git a/packages/web-templates/package.json b/packages/web-templates/package.json index 4f8ff90d9..95475ad27 100644 --- a/packages/web-templates/package.json +++ b/packages/web-templates/package.json @@ -1,6 +1,6 @@ { "name": "@qwen-code/web-templates", - "version": "0.26.16", + "version": "0.26.20", "description": "Web templates bundled as embeddable JS/CSS strings", "repository": { "type": "git", diff --git a/packages/webui/package.json b/packages/webui/package.json index 75068bae6..ff6a68820 100644 --- a/packages/webui/package.json +++ b/packages/webui/package.json @@ -1,6 +1,6 @@ { "name": "@qwen-code/webui", - "version": "0.26.16", + "version": "0.26.20", "description": "Shared UI components for proto packages", "type": "module", "main": "./dist/index.cjs",