fix: recover chats after terminal provider failures

Author: arul28

apps/desktop/src/main/services/chat/agentChatService.test.ts

@@ -13681,6 +13681,125 @@ describe("createAgentChatService", () => {
       }));
     });
 
+    it("emits one Codex failure when error precedes turn/completed with the same payload", async () => {
+      const events: AgentChatEventEnvelope[] = [];
+      const { service } = createService({
+        onEvent: (event: AgentChatEventEnvelope) => events.push(event),
+      });
+      const session = await service.createSession({
+        laneId: "lane-1",
+        provider: "codex",
+        model: "gpt-5.4",
+      });
+
+      const turn = service.runSessionTurn({
+        sessionId: session.id,
+        text: "Continue shipping the fix.",
+      });
+      await vi.waitFor(() => {
+        expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/start")).toBe(true);
+      });
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "turn/started",
+        params: { turn: { id: "turn-capacity", status: "inProgress" } },
+      });
+      const error = {
+        message: "Selected model is at capacity. Please try a different model.",
+        codexErrorInfo: "serverOverloaded",
+      };
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "error",
+        params: { turnId: "turn-capacity", error, willRetry: false },
+      });
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "turn/completed",
+        params: {
+          turn: {
+            id: "turn-capacity",
+            status: "failed",
+            error,
+          },
+        },
+      });
+
+      await expect(turn).resolves.toEqual(expect.objectContaining({ turnId: "turn-capacity" }));
+      expect(events.filter((event) => event.event.type === "error")).toHaveLength(1);
+      expect(events).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          event: expect.objectContaining({
+            type: "status",
+            turnStatus: "failed",
+            turnId: "turn-capacity",
+          }),
+        }),
+        expect.objectContaining({
+          event: expect.objectContaining({
+            type: "done",
+            status: "failed",
+            turnId: "turn-capacity",
+          }),
+        }),
+      ]));
+      await expect(service.getSessionSummary(session.id)).resolves.toEqual(
+        expect.objectContaining({ status: "idle" }),
+      );
+    });
+
+    it("keeps Codex willRetry errors non-terminal until turn/completed", async () => {
+      const events: AgentChatEventEnvelope[] = [];
+      const { service } = createService({
+        onEvent: (event: AgentChatEventEnvelope) => events.push(event),
+      });
+      const session = await service.createSession({
+        laneId: "lane-1",
+        provider: "codex",
+        model: "gpt-5.4",
+      });
+
+      const turn = service.runSessionTurn({ sessionId: session.id, text: "Retry transiently." });
+      await vi.waitFor(() => {
+        expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/start")).toBe(true);
+      });
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "turn/started",
+        params: { turn: { id: "turn-retry", status: "inProgress" } },
+      });
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "error",
+        params: {
+          turnId: "turn-retry",
+          willRetry: true,
+          error: { message: "Temporary upstream failure.", codexErrorInfo: "serverOverloaded" },
+        },
+      });
+
+      expect(events.filter((event) => event.event.type === "error")).toHaveLength(0);
+      expect(events).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          event: expect.objectContaining({
+            type: "system_notice",
+            noticeKind: "provider_health",
+            turnId: "turn-retry",
+          }),
+        }),
+      ]));
+      await expect(service.getSessionSummary(session.id)).resolves.toEqual(
+        expect.objectContaining({ status: "active" }),
+      );
+
+      mockState.emitCodexPayload({
+        jsonrpc: "2.0",
+        method: "turn/completed",
+        params: { turn: { id: "turn-retry", status: "completed" } },
+      });
+      await expect(turn).resolves.toEqual(expect.objectContaining({ turnId: "turn-retry" }));
+    });
+
     it("ignores stale Codex lifecycle notifications from a foreign turn", async () => {
       const events: Array<{ type: string; turnId?: string; text?: string }> = [];
       const { service } = createService({

apps/desktop/src/main/services/chat/agentChatService.ts

@@ -704,6 +704,7 @@ type CodexRuntime = {
   agentMessageScopeByTurn: Map<string, "item" | "turn">;
   agentMessageTextByTurn: Map<string, string>;
   recentNotificationKeys: Set<string>;
+  emittedErrorKeys: Set<string>;
   reconciledItemSignaturesByTurn: Map<string, Set<string>>;
   noFirstEventWatchdog: {
     turnId: string;
@@ -20105,6 +20106,38 @@ export function createAgentChatService(args: {
     persistChatState(managed);
   }
 
+  function emitCodexErrorOnce(
+    managed: ManagedChatSession,
+    runtime: CodexRuntime,
+    input: {
+      turnId?: string | null;
+      message: unknown;
+      errorInfo?: unknown;
+      detail?: unknown;
+    },
+  ): void {
+    const turnId = typeof input.turnId === "string" && input.turnId.trim().length
+      ? input.turnId.trim()
+      : null;
+    const message = String(input.message ?? "Codex app-server error.");
+    const errorInfo = formatCodexErrorInfo(input.errorInfo);
+    const detail = typeof input.detail === "string" && input.detail.trim().length
+      ? input.detail.trim()
+      : undefined;
+    if (turnId) {
+      const semanticKey = JSON.stringify([turnId, message.trim(), errorInfo ?? null, detail ?? null]);
+      if (runtime.emittedErrorKeys.has(semanticKey)) return;
+      rememberBoundedId(runtime.emittedErrorKeys, semanticKey, 512);
+    }
+    emitChatEvent(managed, {
+      type: "error",
+      message,
+      ...(turnId ? { turnId } : {}),
+      ...(errorInfo ? { errorInfo } : {}),
+      ...(detail ? { detail } : {}),
+    });
+  }
+
   async function finishCodexTurnFromReconciledState(
     managed: ManagedChatSession,
     runtime: CodexRuntime,
@@ -20148,11 +20181,11 @@ export function createAgentChatService(args: {
     const error = asRecord(turn.error);
     const errorMessage = stringOrNull(error?.message);
     if (status === "failed" && errorMessage) {
-      emitChatEvent(managed, {
-        type: "error",
-        message: errorMessage,
+      emitCodexErrorOnce(managed, runtime, {
         turnId,
-        errorInfo: formatCodexErrorInfo(error?.codexErrorInfo),
+        message: errorMessage,
+        errorInfo: error?.codexErrorInfo,
+        detail: error?.additionalDetails,
       });
     }
 
@@ -20599,11 +20632,11 @@ export function createAgentChatService(args: {
       }
 
       if (status === "failed" && turn?.error?.message) {
-        emitChatEvent(managed, {
-          type: "error",
-          message: String(turn.error.message),
+        emitCodexErrorOnce(managed, runtime, {
           turnId,
-          errorInfo: formatCodexErrorInfo(turn.error.codexErrorInfo)
+          message: turn.error.message,
+          errorInfo: turn.error.codexErrorInfo,
+          detail: (turn.error as { additionalDetails?: unknown }).additionalDetails,
         });
       }
 
@@ -20997,12 +21030,28 @@ export function createAgentChatService(args: {
     }
 
     if (method === "error") {
-      const error = (params.error as { message?: unknown; codexErrorInfo?: unknown } | null) ?? null;
-      emitChatEvent(managed, {
-        type: "error",
-        message: String(error?.message ?? "Codex app-server error."),
-        turnId: typeof params.turnId === "string" ? params.turnId : undefined,
-        errorInfo: formatCodexErrorInfo(error?.codexErrorInfo)
+      const error = (params.error as {
+        message?: unknown;
+        codexErrorInfo?: unknown;
+        additionalDetails?: unknown;
+      } | null) ?? null;
+      const turnId = typeof params.turnId === "string" ? params.turnId : runtime.activeTurnId ?? undefined;
+      if (params.willRetry === true) {
+        emitChatEvent(managed, {
+          type: "system_notice",
+          noticeKind: "provider_health",
+          severity: "warning",
+          message: "Codex hit a provider error and is retrying automatically.",
+          detail: String(error?.message ?? "The provider request failed."),
+          ...(turnId ? { turnId } : {}),
+        });
+        return;
+      }
+      emitCodexErrorOnce(managed, runtime, {
+        turnId,
+        message: error?.message ?? "Codex app-server error.",
+        errorInfo: error?.codexErrorInfo,
+        detail: error?.additionalDetails,
       });
       return;
     }
@@ -21224,6 +21273,7 @@ export function createAgentChatService(args: {
       agentMessageScopeByTurn: new Map<string, "item" | "turn">(),
       agentMessageTextByTurn: new Map<string, string>(),
       recentNotificationKeys: new Set<string>(),
+      emittedErrorKeys: new Set<string>(),
       reconciledItemSignaturesByTurn: new Map<string, Set<string>>(),
       noFirstEventWatchdog: null,
       stalledTurnIds: new Set<string>(),

apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx

@@ -1051,6 +1051,8 @@ export function AgentChatComposer({
   appControlContextItems = [],
   builtInBrowserContextItems = [],
   modelSelectionLocked = false,
+  modelPickerOpenRequestKey,
+  onModelPickerOpenRequestHandled,
   permissionModeLocked = false,
   hideNativeControls = false,
   orchestrationRole = null,
@@ -1183,6 +1185,8 @@ export function AgentChatComposer({
   builtInBrowserContextItems?: BuiltInBrowserContextItem[];
   executionModeOptions?: ExecutionModeOption[];
   modelSelectionLocked?: boolean;
+  modelPickerOpenRequestKey?: number;
+  onModelPickerOpenRequestHandled?: () => void;
   permissionModeLocked?: boolean;
   hideNativeControls?: boolean;
   /**
@@ -3849,6 +3853,8 @@ export function AgentChatComposer({
                   value={modelId}
                   onChange={onModelChange}
                   surfaceKey="chat-composer"
+                  openRequestKey={modelPickerOpenRequestKey}
+                  onOpenRequestHandled={onModelPickerOpenRequestHandled}
                   {...(availableModelIds ? { availableModelIds } : {})}
                   constrainToAvailableModelIds={constrainModelSelection}
                   {...(providerAuthStatus ? { providerAuthStatus } : {})}

apps/desktop/src/renderer/components/chat/AgentChatMessageList.tsx

@@ -91,6 +91,7 @@ import {
 import { BackgroundFinishChip, SubagentResultCard, SubagentSpawnCard } from "./SubagentActivityCards";
 import { ChatUserMinimap } from "./ChatUserMinimap";
 import { AgentCliAuthCard, type AgentCliAuthCardInfo } from "./AgentCliAuthCard";
+import { classifyProviderFailure, ProviderFailureRecoveryCard } from "./ProviderFailureRecoveryCard";
 import { HighlightedCode } from "./CodeHighlighter";
 import { MosaicCard } from "./MosaicCard";
 import { MOSAIC_FENCE_LANGUAGE } from "../../../shared/chatMosaic";
@@ -2635,6 +2636,8 @@ function renderEvent(
   options?: {
     onApproval?: (itemId: string, decision: AgentChatApprovalDecision, responseText?: string | null, answers?: Record<string, string | string[]>) => void;
     onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise<AgentChatRecoverCodexTurnResult>;
+    onRetryProviderFailure?: (turnId: string | null) => void;
+    onChooseProviderFailureModel?: () => void;
     turnModel?: { label: string; modelId?: string; model?: string } | null;
     surfaceMode?: ChatSurfaceMode;
     surfaceProfile?: ChatSurfaceProfile;
@@ -3617,6 +3620,7 @@ function renderEvent(
     const errorCopyValue = event.detail?.trim().length
       ? `${event.message}\n\n${event.detail}`
       : event.message;
+    const recovery = classifyProviderFailure(event);
     const renderAgentCliAuthCard = () => agentCliInfo ? (
       <AgentCliAuthCard
         agentCli={agentCliInfo}
@@ -3680,10 +3684,24 @@ function renderEvent(
               {event.detail}
             </div>
           ) : null}
+          {recovery ? (
+            <ProviderFailureRecoveryCard
+              recovery={recovery}
+              disabled={Boolean(options?.turnActive)}
+              onRetry={options?.onRetryProviderFailure
+                ? () => options.onRetryProviderFailure?.(event.turnId ?? null)
+                : undefined}
+              onChooseModel={options?.onChooseProviderFailureModel}
+            />
+          ) : null}
           {renderAgentCliAuthCard()}
           {event.errorInfo && !agentCliInfo ? (
-            <div className="mt-2 font-mono text-[length:calc(var(--chat-font-size)*10/14)] text-muted-fg/40">
-              {typeof event.errorInfo === "string" ? event.errorInfo : `${event.errorInfo.provider ? `${event.errorInfo.provider}` : ""}${event.errorInfo.model ? ` / ${event.errorInfo.model}` : ""}`}
+            <div
+              className="mt-2 font-mono text-[length:calc(var(--chat-font-size)*10/14)] text-muted-fg/40"
+              title={typeof event.errorInfo === "string" ? event.errorInfo : undefined}
+            >
+              {recovery?.label
+                ?? (typeof event.errorInfo === "string" ? event.errorInfo : `${event.errorInfo.provider ? `${event.errorInfo.provider}` : ""}${event.errorInfo.model ? ` / ${event.errorInfo.model}` : ""}`)}
             </div>
           ) : null}
         </div>
@@ -4143,6 +4161,8 @@ type EventRowProps = {
   turnEndDurationMs?: number | null;
   onApproval?: (itemId: string, decision: AgentChatApprovalDecision, responseText?: string | null, answers?: Record<string, string | string[]>) => void;
   onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise<AgentChatRecoverCodexTurnResult>;
+  onRetryProviderFailure?: (turnId: string | null) => void;
+  onChooseProviderFailureModel?: () => void;
   surfaceMode?: ChatSurfaceMode;
   surfaceProfile?: ChatSurfaceProfile;
   assistantLabel?: string;
@@ -4176,6 +4196,8 @@ const EventRow = React.memo(function EventRow({
   turnEndDurationMs,
   onApproval,
   onCodexRecovery,
+  onRetryProviderFailure,
+  onChooseProviderFailureModel,
   surfaceMode = "standard",
   surfaceProfile = "standard",
   assistantLabel,
@@ -4243,6 +4265,8 @@ const EventRow = React.memo(function EventRow({
         : renderEvent(envelope as RenderEnvelope, {
             onApproval,
             onCodexRecovery,
+            onRetryProviderFailure,
+            onChooseProviderFailureModel,
             turnModel,
             surfaceMode,
             surfaceProfile,
@@ -4571,6 +4595,8 @@ function AgentChatMessageListMain({
     className,
   onApproval,
   onCodexRecovery,
+  onRetryProviderFailure,
+  onChooseProviderFailureModel,
     surfaceMode = "standard",
   surfaceProfile = "standard",
   assistantLabel,
@@ -4595,6 +4621,8 @@ function AgentChatMessageListMain({
   className?: string;
   onApproval?: (itemId: string, decision: AgentChatApprovalDecision, responseText?: string | null, answers?: Record<string, string | string[]>) => void;
   onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise<AgentChatRecoverCodexTurnResult>;
+  onRetryProviderFailure?: (turnId: string | null) => void;
+  onChooseProviderFailureModel?: () => void;
   surfaceMode?: ChatSurfaceMode;
   surfaceProfile?: ChatSurfaceProfile;
   assistantLabel?: string;
@@ -5397,6 +5425,8 @@ function AgentChatMessageListMain({
           turnEndDurationMs={turnEndDurationMs}
           onApproval={handleApproval}
           onCodexRecovery={onCodexRecovery}
+          onRetryProviderFailure={onRetryProviderFailure}
+          onChooseProviderFailureModel={onChooseProviderFailureModel}
           surfaceMode={surfaceMode}
           surfaceProfile={surfaceProfile}
           assistantLabel={assistantLabel}
@@ -5433,6 +5463,8 @@ function AgentChatMessageListMain({
         turnModel={turnModel}
         onApproval={handleApproval}
         onCodexRecovery={onCodexRecovery}
+        onRetryProviderFailure={onRetryProviderFailure}
+        onChooseProviderFailureModel={onChooseProviderFailureModel}
         surfaceMode={surfaceMode}
         surfaceProfile={surfaceProfile}
         assistantLabel={assistantLabel}
@@ -5458,7 +5490,7 @@ function AgentChatMessageListMain({
         assistantTurnCopy={assistantTurnCopy}
       />
     );
-  }, [activeTurnId, anchoredRowKey, assistantLabel, assistantTurnCopyByRowKey, surfaceMode, surfaceProfile, groupedRows, latestWorkLogIndex, turnModelState, handleApproval, handleMeasure, openWorkspacePath, handleNavigateSuggestion, handleReviewChanges, onCodexRecovery, onInsertDraft, onRevealChatTerminal, onRewindFiles, turnDiffSummaries, respondingApprovalIds, pendingApprovalIds, resolvedInputStates, laneId, sessionId, sessionEnded, runtimeName, mosaic, scrollToRowKey]);
+  }, [activeTurnId, anchoredRowKey, assistantLabel, assistantTurnCopyByRowKey, surfaceMode, surfaceProfile, groupedRows, latestWorkLogIndex, turnModelState, handleApproval, handleMeasure, openWorkspacePath, handleNavigateSuggestion, handleReviewChanges, onCodexRecovery, onRetryProviderFailure, onChooseProviderFailureModel, onInsertDraft, onRevealChatTerminal, onRewindFiles, turnDiffSummaries, respondingApprovalIds, pendingApprovalIds, resolvedInputStates, laneId, sessionId, sessionEnded, runtimeName, mosaic, scrollToRowKey]);
 
   // Compute the bottom spacer height for virtualized mode.
   const bottomSpacerHeight = useMemo(() => {