Skip to content

Commit 39e1c85

Browse files
alex-alecualex-alecu
authored
fix(code-review): disable code review when model not allowed (#3590)
1 parent 5a96aa1 commit 39e1c85

13 files changed

Lines changed: 366 additions & 3 deletions

File tree

‎apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,102 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
534534
);
535535
});
536536

537+
it('infers selected-model-unavailable callbacks as action-required failures', async () => {
538+
const errorMessage =
539+
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}';
540+
mockGetCodeReviewById.mockResolvedValue(makeReview());
541+
542+
const response = await POST(
543+
makeRequest({
544+
status: 'failed',
545+
errorMessage,
546+
}),
547+
makeParams(REVIEW_ID)
548+
);
549+
550+
expect(response.status).toBe(200);
551+
expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith(
552+
expect.objectContaining({
553+
status: 'failed',
554+
errorMessage,
555+
terminalReason: 'selected_model_unavailable',
556+
})
557+
);
558+
expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
559+
REVIEW_ID,
560+
'failed',
561+
expect.objectContaining({
562+
errorMessage,
563+
terminalReason: 'selected_model_unavailable',
564+
})
565+
);
566+
expect(mockUpdateCodeReviewStatusIfNonTerminal).not.toHaveBeenCalled();
567+
expect(mockCreateInfraRetryAttemptIfMissing).not.toHaveBeenCalled();
568+
expect(mockRetryReviewFresh).not.toHaveBeenCalled();
569+
expect(mockDisableCodeReviewForActionRequiredFailure).toHaveBeenCalledWith(
570+
expect.objectContaining({
571+
owner: { type: 'user', id: 'user-1', userId: 'user-1' },
572+
platform: 'github',
573+
reviewId: REVIEW_ID,
574+
reason: 'selected_model_unavailable',
575+
errorMessage,
576+
})
577+
);
578+
expect(mockUpdateCheckRun).toHaveBeenCalledWith(
579+
'inst-1',
580+
'owner',
581+
'repo',
582+
12345,
583+
expect.objectContaining({
584+
conclusion: 'action_required',
585+
output: expect.objectContaining({ title: 'Selected model unavailable' }),
586+
}),
587+
'standard'
588+
);
589+
expect(mockFindKiloReviewComment).not.toHaveBeenCalled();
590+
});
591+
592+
it('infers model-not-allowed callbacks as action-required failures', async () => {
593+
const errorMessage =
594+
'prepareSession failed (400): {"error":{"message":"Not Found: The requested model is not allowed for your team.","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}';
595+
mockGetCodeReviewById.mockResolvedValue(makeReview());
596+
597+
const response = await POST(
598+
makeRequest({
599+
status: 'failed',
600+
errorMessage,
601+
}),
602+
makeParams(REVIEW_ID)
603+
);
604+
605+
expect(response.status).toBe(200);
606+
expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
607+
REVIEW_ID,
608+
'failed',
609+
expect.objectContaining({
610+
errorMessage,
611+
terminalReason: 'selected_model_unavailable',
612+
})
613+
);
614+
expect(mockDisableCodeReviewForActionRequiredFailure).toHaveBeenCalledWith(
615+
expect.objectContaining({
616+
reason: 'selected_model_unavailable',
617+
errorMessage,
618+
})
619+
);
620+
expect(mockUpdateCheckRun).toHaveBeenCalledWith(
621+
'inst-1',
622+
'owner',
623+
'repo',
624+
12345,
625+
expect.objectContaining({
626+
conclusion: 'action_required',
627+
output: expect.objectContaining({ title: 'Selected model unavailable' }),
628+
}),
629+
'standard'
630+
);
631+
});
632+
537633
it('infers GitHub installation and IP allow-list callback failures', async () => {
538634
mockGetCodeReviewById.mockResolvedValue(makeReview());
539635

‎apps/web/src/lib/code-reviews/action-required-shared.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ export const CODE_REVIEW_ACTION_REQUIRED_REASONS = [
44
'github_installation_required',
55
'github_ip_allow_list',
66
'byok_invalid_key',
7+
'selected_model_unavailable',
78
] as const;
89

910
export type CodeReviewActionRequiredReason = (typeof CODE_REVIEW_ACTION_REQUIRED_REASONS)[number];
@@ -61,6 +62,17 @@ const COPY_BY_REASON = {
6162
'Code Reviewer was disabled because the selected BYOK API key is invalid or has been revoked. Update the key or choose another model, then enable Code Reviewer again.',
6263
gitlabDescription: 'BYOK API key needs attention for Code Reviewer',
6364
},
65+
selected_model_unavailable: {
66+
title: 'Code Reviewer needs attention',
67+
description:
68+
'Code Reviewer was disabled because the selected model is not available for cloud agent sessions. Choose an available model, then enable Code Reviewer again.',
69+
recoveryLabel: 'Update Code Reviewer settings',
70+
emailReason: 'The selected model is not available for cloud agent sessions.',
71+
checkTitle: 'Selected model unavailable',
72+
checkSummary:
73+
'Code Reviewer was disabled because the selected model is not available for cloud agent sessions. Choose an available model, then enable Code Reviewer again.',
74+
gitlabDescription: 'Selected model unavailable for Code Reviewer',
75+
},
6476
} satisfies Record<CodeReviewActionRequiredReason, CodeReviewActionRequiredCopy>;
6577

6678
const ACTION_REQUIRED_REASON_SET = new Set<string>(CODE_REVIEW_ACTION_REQUIRED_REASONS);
@@ -91,5 +103,9 @@ export function getCodeReviewActionRequiredRecoveryHref(
91103
return 'mailto:hi@kilocode.ai?subject=GitHub%20IP%20allow%20list%20for%20Code%20Reviewer';
92104
}
93105

106+
if (reason === 'selected_model_unavailable') {
107+
return organizationId ? `/organizations/${organizationId}/code-reviews` : '/code-reviews';
108+
}
109+
94110
return organizationId ? `/organizations/${organizationId}/byok` : '/byok';
95111
}

‎apps/web/src/lib/code-reviews/action-required.test.ts‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@ import { and, eq } from 'drizzle-orm';
1111
import {
1212
classifyCodeReviewActionRequiredFailure,
1313
disableCodeReviewForActionRequiredFailure,
14+
getCodeReviewActionRequiredRecoveryHref,
1415
getCodeReviewActionRequiredState,
1516
} from './action-required';
1617

1718
describe('classifyCodeReviewActionRequiredFailure', () => {
18-
it('classifies GitHub installation, GitHub IP allow-list, and BYOK invalid key failures', () => {
19+
it('classifies GitHub installation, GitHub IP allow-list, BYOK invalid key, and selected model failures', () => {
1920
expect(
2021
classifyCodeReviewActionRequiredFailure(
2122
'GitHub token or active app installation required for this repository (no_installation_found)'
@@ -39,6 +40,30 @@ describe('classifyCodeReviewActionRequiredFailure', () => {
3940
'Although you appear to have the correct authorization credentials, the `acme` organization has an IP allow list enabled, and 192.0.2.1 is not permitted.'
4041
)
4142
).toBe('github_ip_allow_list');
43+
44+
expect(
45+
classifyCodeReviewActionRequiredFailure(
46+
'Selected model is not available for this cloud agent session'
47+
)
48+
).toBe('selected_model_unavailable');
49+
50+
expect(
51+
classifyCodeReviewActionRequiredFailure(
52+
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'
53+
)
54+
).toBe('selected_model_unavailable');
55+
56+
expect(
57+
classifyCodeReviewActionRequiredFailure(
58+
'Not Found: The requested model is not allowed for your team.'
59+
)
60+
).toBe('selected_model_unavailable');
61+
62+
expect(
63+
classifyCodeReviewActionRequiredFailure(
64+
'prepareSession failed (400): {"error":{"message":"Not Found: The requested model is not allowed for your team.","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'
65+
)
66+
).toBe('selected_model_unavailable');
4267
});
4368

4469
it('does not classify unrelated auth, rate-limit, or BYOK quota failures', () => {
@@ -49,6 +74,15 @@ describe('classifyCodeReviewActionRequiredFailure', () => {
4974
classifyCodeReviewActionRequiredFailure('[BYOK] Your account quota is exhausted.')
5075
).toBeNull();
5176
});
77+
78+
it('routes selected model recovery to Code Reviewer settings', () => {
79+
expect(getCodeReviewActionRequiredRecoveryHref('selected_model_unavailable')).toBe(
80+
'/code-reviews'
81+
);
82+
expect(getCodeReviewActionRequiredRecoveryHref('selected_model_unavailable', 'org-1')).toBe(
83+
'/organizations/org-1/code-reviews'
84+
);
85+
});
5286
});
5387

5488
describe('disableCodeReviewForActionRequiredFailure', () => {

‎apps/web/src/lib/code-reviews/action-required.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ const CodeReviewActionRequiredStateSchema = z.object({
3636
emailSentAt: z.string().optional(),
3737
});
3838

39+
const SELECTED_MODEL_UNAVAILABLE_MESSAGE =
40+
'selected model is not available for this cloud agent session';
41+
const REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE =
42+
'the requested model is not allowed for your team';
43+
3944
type AgentConfigWithRuntimeState = {
4045
runtime_state?: Record<string, unknown> | null;
4146
};
@@ -102,6 +107,13 @@ export function classifyCodeReviewActionRequiredFailure(
102107
return 'github_ip_allow_list';
103108
}
104109

110+
if (
111+
normalized.includes(SELECTED_MODEL_UNAVAILABLE_MESSAGE) ||
112+
normalized.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE)
113+
) {
114+
return 'selected_model_unavailable';
115+
}
116+
105117
return null;
106118
}
107119

‎apps/web/src/lib/code-reviews/alerting/detectors.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,12 +236,22 @@ describe('code review alert detectors', () => {
236236
reviewValues({ status: 'failed', terminal_reason: 'github_installation_required' }),
237237
reviewValues({ status: 'failed', terminal_reason: 'github_ip_allow_list' }),
238238
reviewValues({ status: 'failed', terminal_reason: 'byok_invalid_key' }),
239+
reviewValues({ status: 'failed', terminal_reason: 'selected_model_unavailable' }),
239240
...Array.from({ length: 13 }, () => reviewValues()),
240241
]);
241242

242243
await expect(evaluateErrorSpike(db)).resolves.toEqual({ tripped: false });
243244
});
244245

246+
it('excludes selected model unavailable from error-spike counts', async () => {
247+
await insertReviews([
248+
reviewValues({ status: 'failed', terminal_reason: 'selected_model_unavailable' }),
249+
...Array.from({ length: 3 }, () => reviewValues()),
250+
]);
251+
252+
await expect(evaluateErrorSpike(db)).resolves.toEqual({ tripped: false });
253+
});
254+
245255
it('excludes legacy model-not-found failed rows from error-spike counts', async () => {
246256
await insertReviews([
247257
reviewValues({

‎apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,58 @@ describe('tryDispatchPendingReviews', () => {
369369
expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1);
370370
});
371371

372+
it('disables Code Reviewer for selected-model worker status failures', async () => {
373+
const recentTimestamp = minutesAgo(1);
374+
const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner;
375+
const errorMessage =
376+
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"}}';
377+
const agentConfig = await insertAgentConfigForUser();
378+
mockGetAgentConfigForOwner.mockResolvedValue(agentConfig);
379+
mockDispatchReview.mockRejectedValue(
380+
new Error("Dispatch returned terminal status 'failed' for review selected-model-review")
381+
);
382+
mockGetReviewStatus.mockResolvedValue({
383+
reviewId: 'unused',
384+
status: 'failed',
385+
errorMessage,
386+
terminalReason: 'selected_model_unavailable',
387+
});
388+
389+
const [review] = await db
390+
.insert(cloud_agent_code_reviews)
391+
.values(
392+
reviewValues({
393+
owner,
394+
status: 'pending',
395+
createdAt: recentTimestamp,
396+
updatedAt: recentTimestamp,
397+
})
398+
)
399+
.returning({ id: cloud_agent_code_reviews.id });
400+
401+
const result = await tryDispatchPendingReviews({
402+
type: 'user',
403+
id: testUser.id,
404+
userId: testUser.id,
405+
});
406+
407+
const storedReview = await getStoredReview(review.id);
408+
const storedConfig = await db.query.agent_configs.findFirst({
409+
where: eq(agent_configs.id, agentConfig.id),
410+
});
411+
412+
expect(result).toEqual({ dispatched: 1, notDispatched: 0, activeCount: 1 });
413+
expect(storedReview).toEqual(
414+
expect.objectContaining({
415+
status: 'failed',
416+
terminalReason: 'selected_model_unavailable',
417+
errorMessage,
418+
})
419+
);
420+
expect(storedConfig?.is_enabled).toBe(false);
421+
expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1);
422+
});
423+
372424
it('refuses to prepare pending work while action-required state is present', async () => {
373425
const recentTimestamp = minutesAgo(1);
374426
const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner;

‎apps/web/src/routers/admin-code-reviews-router.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,52 @@ describe('adminCodeReviewsRouter', () => {
398398
expect(segmentation.ownershipBreakdown[0]).toMatchObject({ failed: 1 });
399399
});
400400

401+
it('buckets selected-model-unavailable terminal reasons as action required', async () => {
402+
const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner;
403+
const [review] = await db
404+
.insert(cloud_agent_code_reviews)
405+
.values(
406+
reviewValues({
407+
owner,
408+
status: 'failed',
409+
createdAt: timestamp(760),
410+
terminalReason: 'selected_model_unavailable',
411+
errorMessage: 'Selected model is not available for this cloud agent session',
412+
})
413+
)
414+
.returning({ id: cloud_agent_code_reviews.id });
415+
416+
await db.insert(cloud_agent_code_review_attempts).values({
417+
code_review_id: review.id,
418+
attempt_number: 1,
419+
status: 'failed',
420+
terminal_reason: 'selected_model_unavailable',
421+
error_message: 'Selected model is not available for this cloud agent session',
422+
created_at: timestamp(761),
423+
started_at: timestamp(762),
424+
completed_at: timestamp(763),
425+
});
426+
427+
const caller = await createCallerForUser(adminUser.id);
428+
const finalErrors = await caller.admin.codeReviews.getErrorAnalysis(filterInput());
429+
const attemptErrors = await caller.admin.codeReviews.getErrorAnalysis(
430+
filterInput({ retryAccountingMode: 'all_attempts' })
431+
);
432+
433+
expect(finalErrors.categories).toEqual([
434+
expect.objectContaining({ category: 'Action Required', count: 1 }),
435+
]);
436+
expect(finalErrors.details).toEqual([
437+
expect.objectContaining({ category: 'Action Required', count: 1 }),
438+
]);
439+
expect(attemptErrors.categories).toEqual([
440+
expect.objectContaining({ category: 'Action Required', count: 1 }),
441+
]);
442+
expect(attemptErrors.details).toEqual([
443+
expect.objectContaining({ category: 'Action Required', count: 1 }),
444+
]);
445+
});
446+
401447
it('classifies all-attempt model-not-found outcomes as cancellations instead of failures', async () => {
402448
const [review] = await db
403449
.insert(cloud_agent_code_reviews)

‎apps/web/src/routers/admin-code-reviews-router.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attem
7373
* Pattern matching is ordered from most-specific to least-specific.
7474
*/
7575
const errorCategoryExpr = sql<string>`CASE
76-
WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key') THEN 'Action Required'
76+
WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
7777
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%429%' THEN 'Rate Limited'
7878
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_reviews.error_message} LIKE '%timed out%' THEN 'Timeout'
7979
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%context window%' OR ${cloud_agent_code_reviews.error_message} LIKE '%token limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%too large%' OR ${cloud_agent_code_reviews.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'
@@ -87,7 +87,7 @@ const errorCategoryExpr = sql<string>`CASE
8787
END`;
8888

8989
const attemptErrorCategoryExpr = sql<string>`CASE
90-
WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key') THEN 'Action Required'
90+
WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
9191
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%429%' THEN 'Rate Limited'
9292
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%timed out%' THEN 'Timeout'
9393
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%context window%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%token limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%too large%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'

‎packages/db/src/schema-types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,6 +1286,7 @@ export const CODE_REVIEW_TERMINAL_REASONS = [
12861286
'github_installation_required',
12871287
'github_ip_allow_list',
12881288
'byok_invalid_key',
1289+
'selected_model_unavailable',
12891290
'user_cancelled',
12901291
'superseded',
12911292
'interrupted',
@@ -1312,6 +1313,7 @@ export const CODE_REVIEW_BENIGN_TERMINAL_REASONS = [
13121313
'github_installation_required',
13131314
'github_ip_allow_list',
13141315
'byok_invalid_key',
1316+
'selected_model_unavailable',
13151317
'user_cancelled',
13161318
'superseded',
13171319
] as const satisfies readonly CodeReviewTerminalReason[];

0 commit comments

Comments
 (0)