fix(code-review): retry broken sandboxes

apps/web/src/app/api/internal/code-review-sandbox-destroyed/route.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it, jest, beforeEach } from '@jest/globals';
+import type { NextRequest } from 'next/server';
+import type * as retryModule from '@/lib/code-reviews/sandbox-retry';
+
+const mockClaimAndDispatchCodeReviewSandboxRetries = jest.fn() as jest.MockedFunction<
+  typeof retryModule.claimAndDispatchCodeReviewSandboxRetries
+>;
+
+jest.mock('@/lib/config.server', () => ({
+  INTERNAL_API_SECRET: 'test-internal-secret',
+}));
+
+jest.mock('@/lib/code-reviews/sandbox-retry', () => ({
+  claimAndDispatchCodeReviewSandboxRetries: mockClaimAndDispatchCodeReviewSandboxRetries,
+}));
+
+jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() }));
+
+function makeRequest(body: Record<string, unknown>, secret = 'test-internal-secret'): NextRequest {
+  return {
+    headers: { get: (name: string) => (name === 'X-Internal-Secret' ? secret : null) },
+    json: () => Promise.resolve(body),
+  } as unknown as NextRequest;
+}
+
+import type { POST as POSTType } from './route';
+
+let POST: typeof POSTType;
+
+beforeEach(async () => {
+  jest.clearAllMocks();
+  mockClaimAndDispatchCodeReviewSandboxRetries.mockResolvedValue({
+    claimed: 0,
+    dispatchedOwners: 0,
+  });
+  ({ POST } = await import('./route'));
+});
+
+describe('POST /api/internal/code-review-sandbox-destroyed', () => {
+  it('requires internal auth', async () => {
+    const response = await POST(
+      makeRequest(
+        { sandboxId: 'usr-sandbox', phase: 'prepareSession', reason: 'sandbox_500' },
+        'bad'
+      )
+    );
+
+    expect(response.status).toBe(401);
+    expect(mockClaimAndDispatchCodeReviewSandboxRetries).not.toHaveBeenCalled();
+  });
+
+  it('claims and dispatches retries for a destroyed sandbox notification', async () => {
+    mockClaimAndDispatchCodeReviewSandboxRetries.mockResolvedValue({
+      claimed: 3,
+      dispatchedOwners: 2,
+    });
+
+    const response = await POST(
+      makeRequest({
+        sandboxId: 'usr-sandbox',
+        phase: 'prepareSession',
+        reason: 'sandbox_500',
+        destroyedAt: '2026-05-07T12:00:00.000Z',
+      })
+    );
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ claimed: 3, dispatchedOwners: 2 });
+    expect(mockClaimAndDispatchCodeReviewSandboxRetries).toHaveBeenCalledWith({
+      sandboxId: 'usr-sandbox',
+      destroyedAt: '2026-05-07T12:00:00.000Z',
+      source: 'cloud-agent-next-notification',
+    });
+  });
+
+  it('rejects invalid payloads', async () => {
+    const response = await POST(makeRequest({ sandboxId: 'usr-sandbox', reason: 'sandbox_500' }));
+
+    expect(response.status).toBe(400);
+    expect(mockClaimAndDispatchCodeReviewSandboxRetries).not.toHaveBeenCalled();
+  });
+});

apps/web/src/app/api/internal/code-review-sandbox-destroyed/route.ts

@@ -0,0 +1,41 @@
+import type { NextRequest } from 'next/server';
+import { NextResponse } from 'next/server';
+import * as z from 'zod';
+import { INTERNAL_API_SECRET } from '@/lib/config.server';
+import { claimAndDispatchCodeReviewSandboxRetries } from '@/lib/code-reviews/sandbox-retry';
+import { errorExceptInTest } from '@/lib/utils.server';
+import { captureException } from '@sentry/nextjs';
+
+const PayloadSchema = z.object({
+  sandboxId: z.string().min(1),
+  triggeringSessionId: z.string().optional(),
+  phase: z.string().min(1),
+  reason: z.literal('sandbox_500'),
+  destroyedAt: z.string().datetime().optional(),
+});
+
+export async function POST(req: NextRequest) {
+  try {
+    const secret = req.headers.get('X-Internal-Secret');
+    if (!INTERNAL_API_SECRET || secret !== INTERNAL_API_SECRET) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+    }
+
+    const parsed = PayloadSchema.safeParse(await req.json());
+    if (!parsed.success) {
+      return NextResponse.json({ error: 'Invalid payload' }, { status: 400 });
+    }
+
+    const result = await claimAndDispatchCodeReviewSandboxRetries({
+      sandboxId: parsed.data.sandboxId,
+      destroyedAt: parsed.data.destroyedAt,
+      source: 'cloud-agent-next-notification',
+    });
+
+    return NextResponse.json(result);
+  } catch (error) {
+    captureException(error, { tags: { source: 'code-review-sandbox-destroyed' } });
+    errorExceptInTest('[code-review-sandbox-destroyed] Error processing notification', error);
+    return NextResponse.json({ error: 'Failed to process sandbox destruction' }, { status: 500 });
+  }
+}

apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts

@@ -2,15 +2,16 @@ import { describe, expect, it, jest, beforeEach } from '@jest/globals';
 import type { NextRequest } from 'next/server';
 import type * as codeReviewsDbModule from '@/lib/code-reviews/db/code-reviews';
 import type * as platformIntegrationsModule from '@/lib/integrations/db/platform-integrations';
+import type * as sandboxRetryModule from '@/lib/code-reviews/sandbox-retry';
 import type { CloudAgentCodeReview } from '@kilocode/db/schema';
 
 // --- Mock functions ---
 
 const mockGetCodeReviewById = jest.fn() as jest.MockedFunction<
   typeof codeReviewsDbModule.getCodeReviewById
 >;
-const mockUpdateCodeReviewStatus = jest.fn() as jest.MockedFunction<
-  typeof codeReviewsDbModule.updateCodeReviewStatus
+const mockUpdateCodeReviewStatusForAttempt = jest.fn() as jest.MockedFunction<
+  typeof codeReviewsDbModule.updateCodeReviewStatusForAttempt
 >;
 const mockUpdateCodeReviewUsage = jest.fn() as jest.MockedFunction<
   typeof codeReviewsDbModule.updateCodeReviewUsage
@@ -21,6 +22,9 @@ const mockGetSessionUsageFromBilling = jest.fn() as jest.MockedFunction<
 const mockGetIntegrationById = jest.fn() as jest.MockedFunction<
   typeof platformIntegrationsModule.getIntegrationById
 >;
+const mockClaimAndDispatchCodeReviewSandboxRetries = jest.fn() as jest.MockedFunction<
+  typeof sandboxRetryModule.claimAndDispatchCodeReviewSandboxRetries
+>;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const mockTryDispatchPendingReviews = jest.fn<any>();
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -62,11 +66,15 @@ jest.mock('@/lib/config.server', () => ({
 
 jest.mock('@/lib/code-reviews/db/code-reviews', () => ({
   getCodeReviewById: mockGetCodeReviewById,
-  updateCodeReviewStatus: mockUpdateCodeReviewStatus,
+  updateCodeReviewStatusForAttempt: mockUpdateCodeReviewStatusForAttempt,
   updateCodeReviewUsage: mockUpdateCodeReviewUsage,
   getSessionUsageFromBilling: mockGetSessionUsageFromBilling,
 }));
 
+jest.mock('@/lib/code-reviews/sandbox-retry', () => ({
+  claimAndDispatchCodeReviewSandboxRetries: mockClaimAndDispatchCodeReviewSandboxRetries,
+}));
+
 jest.mock('@/lib/integrations/db/platform-integrations', () => ({
   getIntegrationById: mockGetIntegrationById,
 }));
@@ -129,6 +137,7 @@ function makeRequest(body: Record<string, unknown>, secret = VALID_SECRET): Next
     headers: {
       get: (name: string) => (name === 'X-Internal-Secret' ? secret : null),
     },
+    nextUrl: new URL('https://test.kilo.ai/api/internal/code-review-status/test'),
     json: () => Promise.resolve(body),
   } as unknown as NextRequest;
 }
@@ -156,6 +165,11 @@ function makeReview(overrides: Partial<CloudAgentCodeReview> = {}): CloudAgentCo
     platform_project_id: null,
     session_id: null,
     cli_session_id: null,
+    sandbox_id: null,
+    sandbox_retry_count: 0,
+    sandbox_retry_reason: null,
+    sandbox_retry_at: null,
+    current_attempt: 1,
     status: 'running',
     error_message: null,
     terminal_reason: null,
@@ -181,7 +195,11 @@ let POST: typeof POSTType;
 
 beforeEach(async () => {
   jest.clearAllMocks();
-  mockUpdateCodeReviewStatus.mockResolvedValue(undefined);
+  mockUpdateCodeReviewStatusForAttempt.mockResolvedValue(true);
+  mockClaimAndDispatchCodeReviewSandboxRetries.mockResolvedValue({
+    claimed: 0,
+    dispatchedOwners: 0,
+  });
   mockTryDispatchPendingReviews.mockResolvedValue(undefined);
   mockGetBotUserId.mockResolvedValue(null);
   mockGetIntegrationById.mockResolvedValue({
@@ -230,8 +248,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'cancelled',
         expect.objectContaining({ errorMessage: 'User interrupted' })
       );
@@ -250,8 +269,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'failed',
         expect.objectContaining({
           errorMessage: 'Insufficient credits: $1 minimum required',
@@ -272,8 +292,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'failed',
         expect.objectContaining({
           errorMessage: 'This is a paid model. To use paid models, you need to add credits.',
@@ -294,8 +315,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'failed',
         expect.objectContaining({
           errorMessage: 'Add credits to continue, or switch to a free model',
@@ -316,8 +338,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'cancelled',
         expect.objectContaining({
           errorMessage: 'User cancelled the review',
@@ -339,8 +362,9 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
 
       expect(response.status).toBe(200);
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'failed',
         expect.objectContaining({
           terminalReason: 'billing',
@@ -362,24 +386,117 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
         makeParams(REVIEW_ID)
       );
 
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'failed',
         expect.objectContaining({ terminalReason: 'timeout' })
       );
     });
 
+    it('accepts sandbox_error terminalReason', async () => {
+      mockGetCodeReviewById.mockResolvedValue(makeReview());
+
+      await POST(
+        makeRequest({
+          status: 'failed',
+          terminalReason: 'sandbox_error',
+          sandboxId: 'usr-sandbox',
+          errorMessage: 'Sandbox destroyed after 500',
+        }),
+        makeParams(REVIEW_ID)
+      );
+
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
+        REVIEW_ID,
+        1,
+        'failed',
+        expect.objectContaining({
+          terminalReason: 'sandbox_error',
+          sandboxId: 'usr-sandbox',
+        })
+      );
+      expect(mockClaimAndDispatchCodeReviewSandboxRetries).toHaveBeenCalledWith({
+        sandboxId: 'usr-sandbox',
+        source: 'sandbox-error-status-callback',
+      });
+      expect(mockTryDispatchPendingReviews).toHaveBeenCalledWith({
+        type: 'user',
+        id: 'user-1',
+        userId: 'user-1',
+      });
+      expect(mockAddReactionToPR).toHaveBeenCalledWith('inst-1', 'owner', 'repo', 1, 'confused');
+    });
+
+    it('skips terminal cleanup when sandbox retry is claimed', async () => {
+      mockGetCodeReviewById.mockResolvedValue(makeReview());
+      mockClaimAndDispatchCodeReviewSandboxRetries.mockResolvedValue({
+        claimed: 1,
+        dispatchedOwners: 1,
+      });
+
+      await POST(
+        makeRequest({
+          status: 'failed',
+          terminalReason: 'sandbox_error',
+          sandboxId: 'usr-sandbox',
+          errorMessage: 'Sandbox destroyed after 500',
+        }),
+        makeParams(REVIEW_ID)
+      );
+
+      expect(mockClaimAndDispatchCodeReviewSandboxRetries).toHaveBeenCalledWith({
+        sandboxId: 'usr-sandbox',
+        source: 'sandbox-error-status-callback',
+      });
+      expect(mockTryDispatchPendingReviews).not.toHaveBeenCalled();
+      expect(mockAddReactionToPR).not.toHaveBeenCalled();
+    });
+
+    it('persists sandboxId from running callbacks', async () => {
+      mockGetCodeReviewById.mockResolvedValue(makeReview({ status: 'queued' }));
+
+      await POST(
+        makeRequest({ status: 'running', sandboxId: 'usr-sandbox' }),
+        makeParams(REVIEW_ID)
+      );
+
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
+        REVIEW_ID,
+        1,
+        'running',
+        expect.objectContaining({ sandboxId: 'usr-sandbox' })
+      );
+    });
+
     it('handles missing terminalReason gracefully', async () => {
       mockGetCodeReviewById.mockResolvedValue(makeReview());
 
       await POST(makeRequest({ status: 'completed' }), makeParams(REVIEW_ID));
 
-      expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
+      expect(mockUpdateCodeReviewStatusForAttempt).toHaveBeenCalledWith(
         REVIEW_ID,
+        1,
         'completed',
         expect.objectContaining({ terminalReason: undefined })
       );
     });
+
+    it('ignores stale callback attempts before gate updates', async () => {
+      mockGetCodeReviewById.mockResolvedValue(makeReview({ current_attempt: 2 }));
+
+      const response = await POST(
+        makeRequest({ status: 'failed', attempt: 1, errorMessage: 'old failure' }),
+        makeParams(REVIEW_ID)
+      );
+
+      await expect(response.json()).resolves.toMatchObject({
+        success: true,
+        message: 'Stale callback attempt ignored',
+      });
+      expect(mockUpdateCheckRun).not.toHaveBeenCalled();
+      expect(mockUpdateCodeReviewStatusForAttempt).not.toHaveBeenCalled();
+    });
   });
 
   describe('GitHub check run billing messaging', () => {