Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,102 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
);
});

it('infers selected-model-unavailable callbacks as action-required failures', async () => {
const errorMessage =
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}';
mockGetCodeReviewById.mockResolvedValue(makeReview());

const response = await POST(
makeRequest({
status: 'failed',
errorMessage,
}),
makeParams(REVIEW_ID)
);

expect(response.status).toBe(200);
expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith(
expect.objectContaining({
status: 'failed',
errorMessage,
terminalReason: 'selected_model_unavailable',
})
);
expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
REVIEW_ID,
'failed',
expect.objectContaining({
errorMessage,
terminalReason: 'selected_model_unavailable',
})
);
expect(mockUpdateCodeReviewStatusIfNonTerminal).not.toHaveBeenCalled();
expect(mockCreateInfraRetryAttemptIfMissing).not.toHaveBeenCalled();
expect(mockRetryReviewFresh).not.toHaveBeenCalled();
expect(mockDisableCodeReviewForActionRequiredFailure).toHaveBeenCalledWith(
expect.objectContaining({
owner: { type: 'user', id: 'user-1', userId: 'user-1' },
platform: 'github',
reviewId: REVIEW_ID,
reason: 'selected_model_unavailable',
errorMessage,
})
);
expect(mockUpdateCheckRun).toHaveBeenCalledWith(
'inst-1',
'owner',
'repo',
12345,
expect.objectContaining({
conclusion: 'action_required',
output: expect.objectContaining({ title: 'Selected model unavailable' }),
}),
'standard'
);
expect(mockFindKiloReviewComment).not.toHaveBeenCalled();
});

it('infers model-not-allowed callbacks as action-required failures', async () => {
const errorMessage =
'prepareSession failed (400): {"error":{"message":"Not Found: The requested model is not allowed for your team.","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}';
mockGetCodeReviewById.mockResolvedValue(makeReview());

const response = await POST(
makeRequest({
status: 'failed',
errorMessage,
}),
makeParams(REVIEW_ID)
);

expect(response.status).toBe(200);
expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith(
REVIEW_ID,
'failed',
expect.objectContaining({
errorMessage,
terminalReason: 'selected_model_unavailable',
})
);
expect(mockDisableCodeReviewForActionRequiredFailure).toHaveBeenCalledWith(
expect.objectContaining({
reason: 'selected_model_unavailable',
errorMessage,
})
);
expect(mockUpdateCheckRun).toHaveBeenCalledWith(
'inst-1',
'owner',
'repo',
12345,
expect.objectContaining({
conclusion: 'action_required',
output: expect.objectContaining({ title: 'Selected model unavailable' }),
}),
'standard'
);
});

it('infers GitHub installation and IP allow-list callback failures', async () => {
mockGetCodeReviewById.mockResolvedValue(makeReview());

Expand Down
16 changes: 16 additions & 0 deletions apps/web/src/lib/code-reviews/action-required-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export const CODE_REVIEW_ACTION_REQUIRED_REASONS = [
'github_installation_required',
'github_ip_allow_list',
'byok_invalid_key',
'selected_model_unavailable',
] as const;

export type CodeReviewActionRequiredReason = (typeof CODE_REVIEW_ACTION_REQUIRED_REASONS)[number];
Expand Down Expand Up @@ -61,6 +62,17 @@ const COPY_BY_REASON = {
'Code Reviewer was disabled because the selected BYOK API key is invalid or has been revoked. Update the key or choose another model, then enable Code Reviewer again.',
gitlabDescription: 'BYOK API key needs attention for Code Reviewer',
},
selected_model_unavailable: {
title: 'Code Reviewer needs attention',
description:
'Code Reviewer was disabled because the selected model is not available for cloud agent sessions. Choose an available model, then enable Code Reviewer again.',
recoveryLabel: 'Update Code Reviewer settings',
emailReason: 'The selected model is not available for cloud agent sessions.',
checkTitle: 'Selected model unavailable',
checkSummary:
'Code Reviewer was disabled because the selected model is not available for cloud agent sessions. Choose an available model, then enable Code Reviewer again.',
gitlabDescription: 'Selected model unavailable for Code Reviewer',
},
} satisfies Record<CodeReviewActionRequiredReason, CodeReviewActionRequiredCopy>;

const ACTION_REQUIRED_REASON_SET = new Set<string>(CODE_REVIEW_ACTION_REQUIRED_REASONS);
Expand Down Expand Up @@ -91,5 +103,9 @@ export function getCodeReviewActionRequiredRecoveryHref(
return 'mailto:hi@kilocode.ai?subject=GitHub%20IP%20allow%20list%20for%20Code%20Reviewer';
}

if (reason === 'selected_model_unavailable') {
return organizationId ? `/organizations/${organizationId}/code-reviews` : '/code-reviews';
}

return organizationId ? `/organizations/${organizationId}/byok` : '/byok';
}
36 changes: 35 additions & 1 deletion apps/web/src/lib/code-reviews/action-required.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ import { and, eq } from 'drizzle-orm';
import {
classifyCodeReviewActionRequiredFailure,
disableCodeReviewForActionRequiredFailure,
getCodeReviewActionRequiredRecoveryHref,
getCodeReviewActionRequiredState,
} from './action-required';

describe('classifyCodeReviewActionRequiredFailure', () => {
it('classifies GitHub installation, GitHub IP allow-list, and BYOK invalid key failures', () => {
it('classifies GitHub installation, GitHub IP allow-list, BYOK invalid key, and selected model failures', () => {
expect(
classifyCodeReviewActionRequiredFailure(
'GitHub token or active app installation required for this repository (no_installation_found)'
Expand All @@ -39,6 +40,30 @@ describe('classifyCodeReviewActionRequiredFailure', () => {
'Although you appear to have the correct authorization credentials, the `acme` organization has an IP allow list enabled, and 192.0.2.1 is not permitted.'
)
).toBe('github_ip_allow_list');

expect(
classifyCodeReviewActionRequiredFailure(
'Selected model is not available for this cloud agent session'
)
).toBe('selected_model_unavailable');

expect(
classifyCodeReviewActionRequiredFailure(
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'
)
).toBe('selected_model_unavailable');

expect(
classifyCodeReviewActionRequiredFailure(
'Not Found: The requested model is not allowed for your team.'
)
).toBe('selected_model_unavailable');

expect(
classifyCodeReviewActionRequiredFailure(
'prepareSession failed (400): {"error":{"message":"Not Found: The requested model is not allowed for your team.","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'
)
).toBe('selected_model_unavailable');
});

it('does not classify unrelated auth, rate-limit, or BYOK quota failures', () => {
Expand All @@ -49,6 +74,15 @@ describe('classifyCodeReviewActionRequiredFailure', () => {
classifyCodeReviewActionRequiredFailure('[BYOK] Your account quota is exhausted.')
).toBeNull();
});

it('routes selected model recovery to Code Reviewer settings', () => {
expect(getCodeReviewActionRequiredRecoveryHref('selected_model_unavailable')).toBe(
'/code-reviews'
);
expect(getCodeReviewActionRequiredRecoveryHref('selected_model_unavailable', 'org-1')).toBe(
'/organizations/org-1/code-reviews'
);
});
});

describe('disableCodeReviewForActionRequiredFailure', () => {
Expand Down
12 changes: 12 additions & 0 deletions apps/web/src/lib/code-reviews/action-required.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ const CodeReviewActionRequiredStateSchema = z.object({
emailSentAt: z.string().optional(),
});

const SELECTED_MODEL_UNAVAILABLE_MESSAGE =
'selected model is not available for this cloud agent session';
const REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE =
'the requested model is not allowed for your team';

type AgentConfigWithRuntimeState = {
runtime_state?: Record<string, unknown> | null;
};
Expand Down Expand Up @@ -102,6 +107,13 @@ export function classifyCodeReviewActionRequiredFailure(
return 'github_ip_allow_list';
}

if (
normalized.includes(SELECTED_MODEL_UNAVAILABLE_MESSAGE) ||
normalized.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE)
) {
return 'selected_model_unavailable';
}

return null;
}

Expand Down
10 changes: 10 additions & 0 deletions apps/web/src/lib/code-reviews/alerting/detectors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,12 +236,22 @@ describe('code review alert detectors', () => {
reviewValues({ status: 'failed', terminal_reason: 'github_installation_required' }),
reviewValues({ status: 'failed', terminal_reason: 'github_ip_allow_list' }),
reviewValues({ status: 'failed', terminal_reason: 'byok_invalid_key' }),
reviewValues({ status: 'failed', terminal_reason: 'selected_model_unavailable' }),
...Array.from({ length: 13 }, () => reviewValues()),
]);

await expect(evaluateErrorSpike(db)).resolves.toEqual({ tripped: false });
});

it('excludes selected model unavailable from error-spike counts', async () => {
await insertReviews([
reviewValues({ status: 'failed', terminal_reason: 'selected_model_unavailable' }),
...Array.from({ length: 3 }, () => reviewValues()),
]);

await expect(evaluateErrorSpike(db)).resolves.toEqual({ tripped: false });
});

it('excludes legacy model-not-found failed rows from error-spike counts', async () => {
await insertReviews([
reviewValues({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,58 @@ describe('tryDispatchPendingReviews', () => {
expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1);
});

it('disables Code Reviewer for selected-model worker status failures', async () => {
const recentTimestamp = minutesAgo(1);
const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner;
const errorMessage =
'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"}}';
const agentConfig = await insertAgentConfigForUser();
mockGetAgentConfigForOwner.mockResolvedValue(agentConfig);
mockDispatchReview.mockRejectedValue(
new Error("Dispatch returned terminal status 'failed' for review selected-model-review")
);
mockGetReviewStatus.mockResolvedValue({
reviewId: 'unused',
status: 'failed',
errorMessage,
terminalReason: 'selected_model_unavailable',
});

const [review] = await db
.insert(cloud_agent_code_reviews)
.values(
reviewValues({
owner,
status: 'pending',
createdAt: recentTimestamp,
updatedAt: recentTimestamp,
})
)
.returning({ id: cloud_agent_code_reviews.id });

const result = await tryDispatchPendingReviews({
type: 'user',
id: testUser.id,
userId: testUser.id,
});

const storedReview = await getStoredReview(review.id);
const storedConfig = await db.query.agent_configs.findFirst({
where: eq(agent_configs.id, agentConfig.id),
});

expect(result).toEqual({ dispatched: 1, notDispatched: 0, activeCount: 1 });
expect(storedReview).toEqual(
expect.objectContaining({
status: 'failed',
terminalReason: 'selected_model_unavailable',
errorMessage,
})
);
expect(storedConfig?.is_enabled).toBe(false);
expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1);
});

it('refuses to prepare pending work while action-required state is present', async () => {
const recentTimestamp = minutesAgo(1);
const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner;
Expand Down
46 changes: 46 additions & 0 deletions apps/web/src/routers/admin-code-reviews-router.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,52 @@ describe('adminCodeReviewsRouter', () => {
expect(segmentation.ownershipBreakdown[0]).toMatchObject({ failed: 1 });
});

it('buckets selected-model-unavailable terminal reasons as action required', async () => {
const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner;
const [review] = await db
.insert(cloud_agent_code_reviews)
.values(
reviewValues({
owner,
status: 'failed',
createdAt: timestamp(760),
terminalReason: 'selected_model_unavailable',
errorMessage: 'Selected model is not available for this cloud agent session',
})
)
.returning({ id: cloud_agent_code_reviews.id });

await db.insert(cloud_agent_code_review_attempts).values({
code_review_id: review.id,
attempt_number: 1,
status: 'failed',
terminal_reason: 'selected_model_unavailable',
error_message: 'Selected model is not available for this cloud agent session',
created_at: timestamp(761),
started_at: timestamp(762),
completed_at: timestamp(763),
});

const caller = await createCallerForUser(adminUser.id);
const finalErrors = await caller.admin.codeReviews.getErrorAnalysis(filterInput());
const attemptErrors = await caller.admin.codeReviews.getErrorAnalysis(
filterInput({ retryAccountingMode: 'all_attempts' })
);

expect(finalErrors.categories).toEqual([
expect.objectContaining({ category: 'Action Required', count: 1 }),
]);
expect(finalErrors.details).toEqual([
expect.objectContaining({ category: 'Action Required', count: 1 }),
]);
expect(attemptErrors.categories).toEqual([
expect.objectContaining({ category: 'Action Required', count: 1 }),
]);
expect(attemptErrors.details).toEqual([
expect.objectContaining({ category: 'Action Required', count: 1 }),
]);
});

it('classifies all-attempt model-not-found outcomes as cancellations instead of failures', async () => {
const [review] = await db
.insert(cloud_agent_code_reviews)
Expand Down
4 changes: 2 additions & 2 deletions apps/web/src/routers/admin-code-reviews-router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attem
* Pattern matching is ordered from most-specific to least-specific.
*/
const errorCategoryExpr = sql<string>`CASE
WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key') THEN 'Action Required'
WHEN ${cloud_agent_code_reviews.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%429%' THEN 'Rate Limited'
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_reviews.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_reviews.error_message} LIKE '%timed out%' THEN 'Timeout'
WHEN ${cloud_agent_code_reviews.error_message} LIKE '%context window%' OR ${cloud_agent_code_reviews.error_message} LIKE '%token limit%' OR ${cloud_agent_code_reviews.error_message} LIKE '%too large%' OR ${cloud_agent_code_reviews.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'
Expand All @@ -87,7 +87,7 @@ const errorCategoryExpr = sql<string>`CASE
END`;

const attemptErrorCategoryExpr = sql<string>`CASE
WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key') THEN 'Action Required'
WHEN ${cloud_agent_code_review_attempts.terminal_reason} IN ('github_installation_required', 'github_ip_allow_list', 'byok_invalid_key', 'selected_model_unavailable') THEN 'Action Required'
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Rate limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%429%' THEN 'Rate Limited'
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%Timeout%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%ETIMEDOUT%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%timed out%' THEN 'Timeout'
WHEN ${cloud_agent_code_review_attempts.error_message} LIKE '%context window%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%token limit%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%too large%' OR ${cloud_agent_code_review_attempts.error_message} LIKE '%maximum context length%' THEN 'Context Window Exceeded'
Expand Down
2 changes: 2 additions & 0 deletions packages/db/src/schema-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1286,6 +1286,7 @@ export const CODE_REVIEW_TERMINAL_REASONS = [
'github_installation_required',
'github_ip_allow_list',
'byok_invalid_key',
'selected_model_unavailable',
'user_cancelled',
'superseded',
'interrupted',
Expand All @@ -1312,6 +1313,7 @@ export const CODE_REVIEW_BENIGN_TERMINAL_REASONS = [
'github_installation_required',
'github_ip_allow_list',
'byok_invalid_key',
'selected_model_unavailable',
'user_cancelled',
'superseded',
] as const satisfies readonly CodeReviewTerminalReason[];
Expand Down
Loading
Loading