Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,12 @@
"model": "gemini-3.1-flash-lite"
}
},
"gemini-3.5-flash": {

Check warning on line 605 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "chat-base-3",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 608 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"gemma-4-31b-it": {
"extends": "chat-base-3",
"modelConfig": {
Expand All @@ -626,6 +632,12 @@
"model": "gemini-3-flash-preview"
}
},
"gemini-3.5-flash-base": {

Check warning on line 635 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "base",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 638 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"classifier": {
"extends": "base",
"modelConfig": {
Expand Down Expand Up @@ -868,6 +880,16 @@
"multimodalToolUse": true
}
},
"gemini-3.5-flash": {

Check warning on line 883 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"tier": "flash",
"family": "gemini-3",
"isPreview": false,
"isVisible": true,
"features": {
"thinking": false,
"multimodalToolUse": true
}
},
"gemini-2.5-pro": {
"tier": "pro",
"family": "gemini-2.5",
Expand Down Expand Up @@ -1020,9 +1042,46 @@
"contexts": [
{
"condition": {
"hasAccessToPreview": false,
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1048 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false,
"useGemini3_5Flash": false
},
"target": "gemini-2.5-flash"
}
]
},
"gemini-3.5-flash": {

Check warning on line 1059 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"default": "gemini-3.5-flash",

Check warning on line 1060 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"contexts": [
{
"condition": {
"useGemini3_5Flash": false,
"hasAccessToPreview": false
},
"target": "gemini-2.5-flash"
},
{
"condition": {
"useGemini3_5Flash": false
},
"target": "gemini-3-flash-preview"
}
]
},
"gemini-2.5-flash": {
"default": "gemini-2.5-flash",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1084 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
]
},
Expand Down Expand Up @@ -1104,6 +1163,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1170 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false
Expand Down Expand Up @@ -1157,6 +1222,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"
},
{
"condition": {
"hasAccessToPreview": false
Expand Down
28 changes: 24 additions & 4 deletions evals/llm-judge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,30 @@ export class LLMJudge {

for (const res of rawResults) {
// Remove any punctuation the model might have appended
const cleanRes = res.replace(/[^A-Z]/g, '');
if (cleanRes.startsWith('YES')) yes++;
else if (cleanRes.startsWith('NO')) no++;
else other++;
const cleanRes = res.replace(/[^A-Z ]/g, '');
if (
cleanRes.includes('THE ANSWER IS YES') ||
cleanRes.includes('ANSWER IS YES') ||
cleanRes.endsWith('YES')
) {
yes++;
} else if (
cleanRes.includes('THE ANSWER IS NO') ||
cleanRes.includes('ANSWER IS NO') ||
cleanRes.endsWith('NO')
) {
no++;
} else if (cleanRes.trim() === 'YES') {
yes++;
} else if (cleanRes.trim() === 'NO') {
no++;
} else {
// Fallback: look for YES or NO as standalone words or at the end
const words = cleanRes.split(/\s+/);
if (words.includes('YES')) yes++;
else if (words.includes('NO')) no++;
else other++;
}
Comment thread
DavidAPierce marked this conversation as resolved.
}

// Pass if YES > NO and YES > OTHER (plurality)
Expand Down
3 changes: 3 additions & 0 deletions packages/cli/src/acp/acpUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ export function buildAvailableModels(
const preferredModel = config.getModel() || GEMINI_MODEL_ALIAS_AUTO;
const shouldShowPreviewModels = config.getHasAccessToPreviewModel();
const useGemini31 = config.getGemini31LaunchedSync?.() ?? false;
const useGemini3_5Flash = config.hasGemini35FlashGAAccess?.() ?? false;
const selectedAuthType = settings.merged.security.auth.selectedType;
const useCustomToolModel =
useGemini31 && selectedAuthType === AuthType.USE_GEMINI;
Expand All @@ -276,6 +277,7 @@ export function buildAvailableModels(
) {
const options = config.getModelConfigService().getAvailableModelOptions({
useGemini3_1: useGemini31,
useGemini3_5Flash,
useCustomTools: useCustomToolModel,
hasAccessToPreview: shouldShowPreviewModels,
});
Expand All @@ -294,6 +296,7 @@ export function buildAvailableModels(
description: getAutoModelDescription(
shouldShowPreviewModels,
useGemini31,
useGemini3_5Flash,
),
},
];
Expand Down
6 changes: 6 additions & 0 deletions packages/cli/src/ui/components/ModelDialog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {

const shouldShowPreviewModels = config?.getHasAccessToPreviewModel() ?? false;
const useGemini31 = config?.getGemini31LaunchedSync?.() ?? false;
const useGemini3_5Flash = config?.hasGemini35FlashGAAccess?.() ?? false;
const selectedAuthType = settings.merged.security.auth.selectedType;
const useCustomToolModel =
useGemini31 && selectedAuthType === AuthType.USE_GEMINI;
Expand Down Expand Up @@ -129,6 +130,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
.getModelConfigService()
.getAvailableModelOptions({
useGemini3_1: useGemini31,
useGemini3_5Flash,
useCustomTools: useCustomToolModel,
hasAccessToPreview: shouldShowPreviewModels,
hasAccessToProModel,
Expand Down Expand Up @@ -162,6 +164,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
description: getAutoModelDescription(
shouldShowPreviewModels,
useGemini31,
useGemini3_5Flash,
),
key: GEMINI_MODEL_ALIAS_AUTO,
},
Expand All @@ -181,6 +184,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
shouldShowPreviewModels,
manualModelSelected,
useGemini31,
useGemini3_5Flash,
useCustomToolModel,
hasAccessToProModel,
]);
Expand All @@ -195,6 +199,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
.getModelConfigService()
.getAvailableModelOptions({
useGemini3_1: useGemini31,
useGemini3_5Flash,
useCustomTools: useCustomToolModel,
hasAccessToPreview: shouldShowPreviewModels,
hasAccessToProModel,
Expand Down Expand Up @@ -287,6 +292,7 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
}, [
shouldShowPreviewModels,
useGemini31,
useGemini3_5Flash,
useCustomToolModel,
hasAccessToProModel,
config,
Expand Down
43 changes: 43 additions & 0 deletions packages/cli/src/ui/components/ModelStatsDisplay.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,49 @@ describe('<ModelStatsDisplay />', () => {
unmount();
});

it('should resolve gemini-3-flash to gemini-3.5-flash via getDisplayString', async () => {
const { lastFrame, unmount } = await renderWithMockedStats({
models: {
'gemini-3-flash': {
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
tokens: {
input: 5,
prompt: 10,
candidates: 20,
total: 30,
cached: 5,
thoughts: 2,
tool: 1,
},
roles: {},
},
},
tools: {
totalCalls: 0,
totalSuccess: 0,
totalFail: 0,
totalDurationMs: 0,
totalDecisions: {
accept: 0,
reject: 0,
modify: 0,
[ToolCallDecision.AUTO_ACCEPT]: 0,
},
byName: {},
},
files: {
totalLinesAdded: 0,
totalLinesRemoved: 0,
},
});

const output = lastFrame();
expect(output).toContain('gemini-3.5-flash');
expect(output).not.toContain('gemini-3-flash');
expect(output).toMatchSnapshot();
unmount();
});

it('should handle models with long names (gemini-3-*-preview) without layout breaking', async () => {
const { lastFrame, unmount } = await renderWithMockedStats(
{
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/ui/components/ModelStatsDisplay.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ export const ModelStatsDisplay: React.FC<ModelStatsDisplayProps> = ({
},
...modelNames.map((name) => ({
key: name,
header: name,
header: getDisplayString(name),
flexGrow: 1,
renderCell: (row: StatRowData) => {
// Don't render anything for section headers in model columns
Expand Down
27 changes: 27 additions & 0 deletions packages/cli/src/ui/components/StatsDisplay.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,33 @@ describe('<StatsDisplay />', () => {
expect(output).toMatchSnapshot();
});

it('resolves gemini-3-flash to gemini-3.5-flash in the model usage table', async () => {
const metrics = createTestMetrics({
models: {
'gemini-3-flash': {
api: { totalRequests: 5, totalErrors: 0, totalLatencyMs: 3000 },
tokens: {
input: 1000,
prompt: 2000,
candidates: 3000,
total: 5000,
cached: 500,
thoughts: 100,
tool: 50,
},
roles: {},
},
},
});

const { lastFrame } = await renderWithMockedStats(metrics);
const output = lastFrame();

expect(output).toContain('gemini-3.5-flash');
expect(output).not.toContain('gemini-3-flash\u0020'); // Avoid matching parts of substrings if not intended
expect(output).toMatchSnapshot();
});

it('renders role breakdown correctly under models', async () => {
const metrics = createTestMetrics({
models: {
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/ui/components/StatsDisplay.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import {
import { computeSessionStats } from '../utils/computeStats.js';
import { useSettings } from '../contexts/SettingsContext.js';
import type { QuotaStats } from '../types.js';
import { LlmRole } from '@google/gemini-cli-core';
import { LlmRole, getDisplayString } from '@google/gemini-cli-core';

// A more flexible and powerful StatRow component
interface StatRowProps {
Expand Down Expand Up @@ -101,7 +101,7 @@ const ModelUsageTable: React.FC<ModelUsageTableProps> = ({ models }) => {
Object.entries(models).forEach(([name, metrics]) => {
rows.push({
name,
displayName: name,
displayName: getDisplayString(name),
requests: metrics.api.totalRequests,
cachedTokens: metrics.tokens.cached.toLocaleString(),
inputTokens: metrics.tokens.prompt.toLocaleString(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,26 @@ exports[`<ModelStatsDisplay /> > should render "no API calls" message when there
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
"
`;

exports[`<ModelStatsDisplay /> > should resolve gemini-3-flash to gemini-3.5-flash via getDisplayString 1`] = `
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ Model Stats For Nerds │
│ │
│ │
│ Metric gemini-3.5-flash │
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
│ API │
│ Requests 1 │
│ Errors 0 (0.0%) │
│ Avg Latency 100ms │
│ Tokens │
│ Total 30 │
│ ↳ Input 5 │
│ ↳ Cache Reads 5 (50.0%) │
│ ↳ Thoughts 2 │
│ ↳ Tool 1 │
│ ↳ Output 20 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
"
`;
Original file line number Diff line number Diff line change
Expand Up @@ -292,3 +292,30 @@ exports[`<StatsDisplay /> > renders role breakdown correctly under models 1`] =
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
"
`;

exports[`<StatsDisplay /> > resolves gemini-3-flash to gemini-3.5-flash in the model usage table 1`] = `
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ Session Stats │
│ │
│ Interaction Summary │
│ Session ID: test-session-id │
│ Tool Calls: 0 ( ✓ 0 x 0 ) │
│ Success Rate: 0.0% │
│ │
│ Performance │
│ Wall Time: 1s │
│ Agent Active: 3.0s │
│ » API Time: 3.0s (100.0%) │
│ » Tool Time: 0s (0.0%) │
│ │
│ │
│ Model Usage │
│ Use /model to view model quota information │
│ │
│ Model Reqs Input Tokens Cache Reads Output Tokens │
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
│ gemini-3.5-flash 5 2,000 500 3,000 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
"
`;
Loading
Loading