Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ import { program } from './cli.js';
// In web mode, we should NOT exit on transient errors — log and continue
const isWebMode = process.argv.includes('web');

// COD-115: Codeman IS a tmux controller; it must never present as a tmux *client*.
// If the web server is launched from inside a tmux pane it inherits TMUX/TMUX_PANE,
// and tmux's nesting guard then kills every new attach-bridge PTY (exit 1 → respawn
// loop, crash-looping any new tmux-backed session). Scrub at the root so every
// downstream `{...process.env}` spread (attach, send-keys, create) is clean regardless
// of launch context. `delete` (not `= undefined`, which node-pty serializes as the
// literal string "undefined" and fails to clear).
if (isWebMode) {
delete process.env.TMUX;
delete process.env.TMUX_PANE;
}

import { MAX_CONSECUTIVE_ERRORS, ERROR_RESET_MS } from './config/server-timing.js';

// Track consecutive unhandled errors in web mode — restart after too many
Expand Down
23 changes: 19 additions & 4 deletions src/session-cli-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,17 +124,32 @@ export function buildClaudeEnv(sessionId: string): Record<string, string | undef
* Lighter than buildClaudeEnv — no PATH augmentation or Codeman vars needed
* since the mux session already has those set.
*
* @param truecolorEnabled - When true, set COLORTERM=truecolor (COD-75 opt-in);
* otherwise leave COLORTERM unset. Mirrors buildEnvExports() so both paths agree.
* @returns Environment variables object for pty.spawn
*/
export function buildMuxAttachEnv(): Record<string, string | undefined> {
return {
export function buildMuxAttachEnv(truecolorEnabled?: boolean): Record<string, string | undefined> {
const env: Record<string, string | undefined> = {
...process.env,
LANG: 'en_US.UTF-8',
LC_ALL: 'en_US.UTF-8',
TERM: 'xterm-256color',
COLORTERM: undefined,
CLAUDECODE: undefined,
};
// COD-115: keys to UNSET must be `delete`d, NOT set to `undefined`. On a
// `{...process.env}` spread the key stays present with value undefined, and node-pty
// serializes it as the literal string "TMUX=undefined" — a non-empty value that still
// trips tmux's nesting guard, killing the attach-bridge PTY (exit 1 → respawn loop).
// The server can be launched from inside tmux; attach clients must never inherit that
// parent tmux context. (Same fix the working create path uses in tmux-manager.ts.)
delete env.TMUX;
delete env.TMUX_PANE;
delete env.CLAUDECODE;
if (truecolorEnabled) {
env.COLORTERM = 'truecolor';
} else {
delete env.COLORTERM; // COD-75: unset for non-truecolor (was `: undefined`, same node-pty quirk)
}
return env;
}

/**
Expand Down
102 changes: 102 additions & 0 deletions src/session-pty-exit-breaker.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/**
* @fileoverview Circuit breaker bounding repeated non-zero interactive-PTY exits (COD-118).
*
* Defense-in-depth after COD-115: if the interactive PTY exits non-zero repeatedly,
* external recovery/reconnect paths recreate it indefinitely (COD-115 observed 114
* `exited with code: 1` events + orphan sessions). This breaker tracks recent
* non-zero exits within a sliding window and "trips" once they exceed a threshold,
* so the Session can refuse to respawn and surface an error state instead of looping.
*
* Design notes:
* - PURE + dependency-free. Time is INJECTED (`nowMs` passed to `recordExit`); the
* breaker never calls `Date.now()` itself, so trip/window logic is deterministically
* unit-testable with no real timers.
* - A clean (exit code 0) exit resets the counter — a session that exited normally is
* not on a crash-loop. (It does NOT clear an already-tripped breaker; only an explicit
* `reset()` — e.g. a user-initiated restart — does that.)
* - Once tripped, stays tripped until `reset()`.
*
* @consumedby session (instantiates one per session; records exits in the interactive
* PTY `onExit` handler; gates `startInteractive()` when tripped; `reset()` on restart)
* @module session-pty-exit-breaker
*/

/** Non-zero interactive-PTY exits within the window required to trip the breaker. */
export const DEFAULT_BREAKER_THRESHOLD = 5;

/** Sliding window (ms) over which non-zero exits accumulate toward the threshold. */
export const DEFAULT_BREAKER_WINDOW_MS = 10_000;

export interface InteractivePtyExitBreakerOptions {
/** Trip after this many non-zero exits within `windowMs` (default 5). */
threshold?: number;
/** Sliding window length in ms (default 10_000). */
windowMs?: number;
}

export interface RecordExitResult {
/** True once the breaker has tripped (stays true until `reset()`). */
tripped: boolean;
/** Number of non-zero exits currently inside the window. */
count: number;
}

/**
* Sliding-window counter that trips on rapid repeated non-zero exits.
*
* 5 within 10s safely clears normal usage (a single exit, an intentional restart)
* while tripping fast on a real loop — COD-115 saw 114 exits, far above 5.
*/
export class InteractivePtyExitBreaker {
private readonly _threshold: number;
private readonly _windowMs: number;

/** Timestamps (ms, injected) of recent non-zero exits, oldest first. */
private _exitTimes: number[] = [];

private _tripped = false;

constructor(opts: InteractivePtyExitBreakerOptions = {}) {
this._threshold = opts.threshold ?? DEFAULT_BREAKER_THRESHOLD;
this._windowMs = opts.windowMs ?? DEFAULT_BREAKER_WINDOW_MS;
}

/** Whether the breaker has tripped (respawn should be blocked). */
get tripped(): boolean {
return this._tripped;
}

/**
* Record a PTY exit. A zero (clean) exit resets the non-zero counter; a non-zero
* exit is added to the window, stale entries are evicted, and the breaker trips
* once the in-window count reaches the threshold.
*
* @param exitCode the PTY exit code (0 = clean)
* @param nowMs injected current time in ms (never read from a real clock)
*/
recordExit(exitCode: number, nowMs: number): RecordExitResult {
if (exitCode === 0) {
// Clean exit: a normal stop, not a crash-loop. Clear accumulated non-zero
// exits. Does NOT un-trip an already-tripped breaker (only reset() does).
this._exitTimes = [];
return { tripped: this._tripped, count: 0 };
}

// Evict exits strictly older than the window, then record this one.
const cutoff = nowMs - this._windowMs;
this._exitTimes = this._exitTimes.filter((t) => t > cutoff);
this._exitTimes.push(nowMs);

if (this._exitTimes.length >= this._threshold) {
this._tripped = true;
}

return { tripped: this._tripped, count: this._exitTimes.length };
}

/** Clear the tripped state and the non-zero counter (e.g. on intentional restart). */
reset(): void {
this._exitTimes = [];
this._tripped = false;
}
}
49 changes: 49 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ import {
import { SessionAutoOps } from './session-auto-ops.js';
import { detectUsageLimitPause } from './usage-limit-patterns.js';
import { SessionTaskCache } from './session-task-cache.js';
import { InteractivePtyExitBreaker } from './session-pty-exit-breaker.js';
import { parseAttachmentMagicLinks } from './attachment-magic.js';
import {
sanitizeAttachmentHistory,
Expand Down Expand Up @@ -288,6 +289,13 @@ export class Session extends EventEmitter {
private _pid: number | null = null;
private _status: SessionStatus = 'idle';
private _currentTaskId: string | null = null;

// COD-118: bound repeated non-zero interactive-PTY exits. Recorded in the
// interactive PTY onExit handler; when it trips, the session flips to 'error'
// and startInteractive() refuses to respawn until an explicit user restart
// calls resetRespawnBreaker(). Defense-in-depth over the COD-115 crash-loop.
private readonly _ptyExitBreaker = new InteractivePtyExitBreaker();
private _respawnBlocked = false;
// Use BufferAccumulator for hot-path buffers to reduce GC pressure
private _terminalBuffer = new BufferAccumulator(MAX_TERMINAL_BUFFER_SIZE, TERMINAL_BUFFER_TRIM_SIZE);
private _textOutput = new BufferAccumulator(MAX_TEXT_OUTPUT_SIZE, TEXT_OUTPUT_TRIM_SIZE);
Expand Down Expand Up @@ -1256,6 +1264,16 @@ export class Session extends EventEmitter {
throw new Error('Session already has a running process');
}

// COD-118: if the PTY exit breaker has tripped (repeated non-zero exits in a
// short window), refuse to respawn. This is the uniform choke point that stops
// automatic recovery/reconnect callers from re-creating a crash-looping PTY.
// An explicit user restart clears it via resetRespawnBreaker().
if (this._respawnBlocked) {
throw new Error(
'Respawn blocked: interactive PTY exited non-zero too many times in a short window (circuit breaker tripped). Restart the session to clear it.'
);
}

this._resetBuffers();

const modeLabel = getModeLabel(this.mode);
Expand Down Expand Up @@ -1496,6 +1514,9 @@ export class Session extends EventEmitter {

this.ptyProcess.onExit(({ exitCode }) => {
console.log('[Session] Interactive PTY exited with code:', exitCode);
// COD-118: record the exit in the circuit breaker BEFORE status bookkeeping.
// A clean (0) exit resets the counter; rapid non-zero repeats trip it.
const breakerResult = this._ptyExitBreaker.recordExit(exitCode, Date.now());
this.ptyProcess = null;
this._pid = null;
this._status = 'idle';
Expand Down Expand Up @@ -1523,10 +1544,38 @@ export class Session extends EventEmitter {
if (this._muxSession && this._mux) {
this._mux.setAttached(this.id, false);
}
// COD-118: if the breaker tripped, surface an error state and block the NEXT
// respawn so recovery/reconnect callers stop looping. Still emit 'exit' below
// for normal cleanup. Cleared by an explicit user restart (resetRespawnBreaker()).
if (breakerResult.tripped && !this._respawnBlocked) {
this._respawnBlocked = true;
this._status = 'error';
console.error(
`[Session] PTY exit circuit breaker tripped for ${this.id} (${breakerResult.count} non-zero exits within window); blocking respawn.`
);
this.emit('respawnBreakerTripped', { count: breakerResult.count });
}
this.emit('exit', exitCode);
});
}

/**
* Clear the interactive-PTY exit circuit breaker (COD-118).
*
* Called on an EXPLICIT, user-initiated (re)start so an intentional restart is
* never blocked by a prior crash-loop trip. Automatic recovery/reconnect paths
* must NOT call this — that's the whole point of the breaker.
*/
resetRespawnBreaker(): void {
this._ptyExitBreaker.reset();
this._respawnBlocked = false;
}

/** Whether the interactive-PTY exit circuit breaker is currently tripped (COD-118). */
get respawnBlocked(): boolean {
return this._respawnBlocked;
}

/**
* Process expensive parsers (ANSI strip, Ralph, bash tool, token, CLI info, task descriptions).
* Called on a throttled schedule (every EXPENSIVE_PROCESS_INTERVAL_MS) instead of on every
Expand Down
10 changes: 10 additions & 0 deletions src/web/public/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ const _SSE_HANDLER_MAP = [
[SSE_EVENTS.SESSION_LIMIT_PAUSE_SCHEDULED, '_onSessionLimitPauseScheduled'],
[SSE_EVENTS.SESSION_LIMIT_RESUME, '_onSessionLimitResume'],
[SSE_EVENTS.SESSION_LIMIT_RESUME_CANCELLED, '_onSessionLimitResumeCancelled'],
[SSE_EVENTS.SESSION_RESPAWN_BREAKER_TRIPPED, '_onSessionRespawnBreakerTripped'],
[SSE_EVENTS.SESSION_CLI_INFO, '_onSessionCliInfo'],
[SSE_EVENTS.SESSION_STATUS_TELEMETRY, '_onSessionStatusTelemetry'],

Expand Down Expand Up @@ -1852,6 +1853,15 @@ class CodemanApp {
this.updateAutoResumeStatus(data.sessionId);
}

// COD-118: the interactive PTY exit circuit breaker tripped (repeated non-zero exits).
// The errored status itself arrives via session:updated; this just surfaces a toast for
// diagnostic clarity so a silently-looping session is obvious. Restart clears the breaker.
_onSessionRespawnBreakerTripped(data) {
const session = this.sessions.get(data.sessionId);
const label = session?.name || 'Session';
this.showToast?.(`${label} stopped: repeated crashes detected. Restart to retry.`, 'error');
}

_onSessionCliInfo(data) {
const session = this.sessions.get(data.sessionId);
if (session) {
Expand Down
1 change: 1 addition & 0 deletions src/web/public/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ const SSE_EVENTS = {
SESSION_LIMIT_PAUSE_SCHEDULED: 'session:limitPauseScheduled',
SESSION_LIMIT_RESUME: 'session:limitResume',
SESSION_LIMIT_RESUME_CANCELLED: 'session:limitResumeCancelled',
SESSION_RESPAWN_BREAKER_TRIPPED: 'session:respawnBreakerTripped',
SESSION_CLI_INFO: 'session:cliInfo',
SESSION_MESSAGE: 'session:message',
SESSION_INTERACTIVE: 'session:interactive',
Expand Down
3 changes: 3 additions & 0 deletions src/web/routes/session-routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,9 @@ export function registerSessionRoutes(
}
}

// COD-118: an explicit user-initiated start clears any tripped PTY-exit
// circuit breaker so an intentional restart is never blocked by a prior crash-loop.
session.resetRespawnBreaker();
await session.startInteractive();
getLifecycleLog().log({
event: 'started',
Expand Down
24 changes: 24 additions & 0 deletions src/web/session-listener-wiring.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export interface SessionListenerRefs {
limitPauseScheduled: (data: { resetAt: number; resumeAt: number; matched: string }) => void;
limitResume: (data: { attempt: number }) => void;
limitResumeCancelled: (data: { reason: string }) => void;
respawnBreakerTripped: (data: { count: number }) => void;
cliInfoUpdated: (data: { version?: string; model?: string; accountType?: string; latestVersion?: string }) => void;
ralphLoopUpdate: (state: RalphTrackerState) => void;
ralphTodoUpdate: (todos: RalphTodoItem[]) => void;
Expand Down Expand Up @@ -270,6 +271,27 @@ export function createSessionListeners(session: Session, deps: SessionListenerDe
deps.persistSessionState(session);
},

/**
* Broadcasts `session:respawnBreakerTripped` (COD-118) — repeated non-zero PTY exits
* tripped the circuit breaker; the session is now errored and respawn is blocked.
* Also pushes the errored state (`session:updated`) so the tab renders the error,
* persists it, and notifies for diagnostic visibility.
*/
respawnBreakerTripped: (data: { count: number }) => {
deps.broadcast(SseEvent.SessionRespawnBreakerTripped, { sessionId: session.id, ...data });
deps.broadcast(SseEvent.SessionUpdated, deps.getSessionStateWithRespawn(session));
deps.persistSessionState(session);
deps.sendPushNotifications(SseEvent.SessionRespawnBreakerTripped, {
sessionId: session.id,
sessionName: session.name,
count: data.count,
});
const tracker = deps.getRunSummaryTracker(session.id);
if (tracker) {
tracker.recordError('Respawn circuit breaker tripped', `${data.count} non-zero PTY exits within window`);
}
},

// ─── CLI Info ────────────────────────────────────────────

/** Broadcasts `session:cliInfo` — Claude Code version, model, account type parsed from terminal */
Expand Down Expand Up @@ -387,6 +409,7 @@ export function attachSessionListeners(session: Session, refs: SessionListenerRe
session.on('limitPauseScheduled', refs.limitPauseScheduled);
session.on('limitResume', refs.limitResume);
session.on('limitResumeCancelled', refs.limitResumeCancelled);
session.on('respawnBreakerTripped', refs.respawnBreakerTripped);
session.on('cliInfoUpdated', refs.cliInfoUpdated);
session.on('ralphLoopUpdate', refs.ralphLoopUpdate);
session.on('ralphTodoUpdate', refs.ralphTodoUpdate);
Expand Down Expand Up @@ -420,6 +443,7 @@ export function detachSessionListeners(session: Session, refs: SessionListenerRe
session.off('limitPauseScheduled', refs.limitPauseScheduled);
session.off('limitResume', refs.limitResume);
session.off('limitResumeCancelled', refs.limitResumeCancelled);
session.off('respawnBreakerTripped', refs.respawnBreakerTripped);
session.off('cliInfoUpdated', refs.cliInfoUpdated);
session.off('ralphLoopUpdate', refs.ralphLoopUpdate);
session.off('ralphTodoUpdate', refs.ralphTodoUpdate);
Expand Down
3 changes: 3 additions & 0 deletions src/web/sse-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ export const SessionLimitPauseScheduled = 'session:limitPauseScheduled' as const
export const SessionLimitResume = 'session:limitResume' as const;
/** Pending usage-limit auto-resume cancelled (session resumed or feature disabled). */
export const SessionLimitResumeCancelled = 'session:limitResumeCancelled' as const;
/** Interactive-PTY exit circuit breaker tripped (COD-118): repeated non-zero exits; respawn blocked, session errored. */
export const SessionRespawnBreakerTripped = 'session:respawnBreakerTripped' as const;
/** CLI version/model info detected from session output. */
export const SessionCliInfo = 'session:cliInfo' as const;
/** General session message (e.g. status text). */
Expand Down Expand Up @@ -384,6 +386,7 @@ export const SseEvent = {
SessionLimitPauseScheduled,
SessionLimitResume,
SessionLimitResumeCancelled,
SessionRespawnBreakerTripped,
SessionCliInfo,
SessionMessage,
SessionInteractive,
Expand Down
3 changes: 3 additions & 0 deletions test/mocks/mock-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,9 @@ export class MockSession extends EventEmitter {
/** Stub for startInteractive */
startInteractive = vi.fn(async () => {});

/** Stub for resetRespawnBreaker (COD-118) */
resetRespawnBreaker = vi.fn();

/** Stub for startShell */
startShell = vi.fn(async () => {});

Expand Down
Loading