Transition to flash GA model when experiment flag is present. (google-gemini#27570)

Author: DavidAPierce

docs/reference/configuration.md

@@ -602,6 +602,12 @@ their corresponding top-level category object in your `settings.json` file.
           "model": "gemini-3.1-flash-lite"
         }
       },
+      "gemini-3.5-flash": {
+        "extends": "chat-base-3",
+        "modelConfig": {
+          "model": "gemini-3.5-flash"
+        }
+      },
       "gemma-4-31b-it": {
         "extends": "chat-base-3",
         "modelConfig": {
@@ -626,6 +632,12 @@ their corresponding top-level category object in your `settings.json` file.
           "model": "gemini-3-flash-preview"
         }
       },
+      "gemini-3.5-flash-base": {
+        "extends": "base",
+        "modelConfig": {
+          "model": "gemini-3.5-flash"
+        }
+      },
       "classifier": {
         "extends": "base",
         "modelConfig": {
@@ -868,6 +880,16 @@ their corresponding top-level category object in your `settings.json` file.
           "multimodalToolUse": true
         }
       },
+      "gemini-3.5-flash": {
+        "tier": "flash",
+        "family": "gemini-3",
+        "isPreview": false,
+        "isVisible": true,
+        "features": {
+          "thinking": false,
+          "multimodalToolUse": true
+        }
+      },
       "gemini-2.5-pro": {
         "tier": "pro",
         "family": "gemini-2.5",
@@ -1020,9 +1042,44 @@ their corresponding top-level category object in your `settings.json` file.
         "contexts": [
           {
             "condition": {
+              "useGemini3_5Flash": true
+            },
+            "target": "gemini-3.5-flash"
+          },
+          {
+            "condition": {
+              "hasAccessToPreview": false
+            },
+            "target": "gemini-2.5-flash"
+          }
+        ]
+      },
+      "gemini-3.5-flash": {
+        "default": "gemini-3.5-flash",
+        "contexts": [
+          {
+            "condition": {
+              "useGemini3_5Flash": false,
               "hasAccessToPreview": false
             },
             "target": "gemini-2.5-flash"
+          },
+          {
+            "condition": {
+              "useGemini3_5Flash": false
+            },
+            "target": "gemini-3-flash-preview"
+          }
+        ]
+      },
+      "gemini-2.5-flash": {
+        "default": "gemini-2.5-flash",
+        "contexts": [
+          {
+            "condition": {
+              "useGemini3_5Flash": true
+            },
+            "target": "gemini-3.5-flash"
           }
         ]
       },
@@ -1104,6 +1161,12 @@ their corresponding top-level category object in your `settings.json` file.
       "flash": {
         "default": "gemini-3-flash-preview",
         "contexts": [
+          {
+            "condition": {
+              "useGemini3_5Flash": true
+            },
+            "target": "gemini-3.5-flash"
+          },
           {
             "condition": {
               "hasAccessToPreview": false
@@ -1157,6 +1220,12 @@ their corresponding top-level category object in your `settings.json` file.
       "flash": {
         "default": "gemini-3-flash-preview",
         "contexts": [
+          {
+            "condition": {
+              "useGemini3_5Flash": true
+            },
+            "target": "gemini-3.5-flash"
+          },
           {
             "condition": {
               "hasAccessToPreview": false

evals/llm-judge.ts

@@ -76,10 +76,30 @@ export class LLMJudge {
 
     for (const res of rawResults) {
       // Remove any punctuation the model might have appended
-      const cleanRes = res.replace(/[^A-Z]/g, '');
-      if (cleanRes.startsWith('YES')) yes++;
-      else if (cleanRes.startsWith('NO')) no++;
-      else other++;
+      const cleanRes = res.replace(/[^A-Z ]/g, '');
+      if (
+        cleanRes.includes('THE ANSWER IS YES') ||
+        cleanRes.includes('ANSWER IS YES') ||
+        cleanRes.endsWith('YES')
+      ) {
+        yes++;
+      } else if (
+        cleanRes.includes('THE ANSWER IS NO') ||
+        cleanRes.includes('ANSWER IS NO') ||
+        cleanRes.endsWith('NO')
+      ) {
+        no++;
+      } else if (cleanRes.trim() === 'YES') {
+        yes++;
+      } else if (cleanRes.trim() === 'NO') {
+        no++;
+      } else {
+        // Fallback: look for YES or NO as standalone words or at the end
+        const words = cleanRes.split(/\s+/);
+        if (words.includes('YES')) yes++;
+        else if (words.includes('NO')) no++;
+        else other++;
+      }
     }
 
     // Pass if YES > NO and YES > OTHER (plurality)

packages/core/src/availability/policyCatalog.ts

@@ -33,6 +33,7 @@ export interface ModelPolicyOptions {
   useGemini31?: boolean;
   useGemini31FlashLite?: boolean;
   useCustomToolModel?: boolean;
+  useGemini3_5Flash?: boolean;
 }
 
 const DEFAULT_ACTIONS: ModelPolicyActionMap = {
@@ -94,6 +95,9 @@ export function getModelPolicyChain(
       PREVIEW_GEMINI_MODEL,
       options.useGemini31,
       options.useCustomToolModel,
+      true,
+      undefined,
+      options.useGemini3_5Flash,
     );
     return [
       definePolicy({

packages/core/src/availability/policyHelpers.ts

@@ -54,6 +54,7 @@ export function resolvePolicyChain(
   const useGemini31 = config.getGemini31LaunchedSync?.() ?? false;
   const useCustomToolModel = config.getUseCustomToolModelSync?.() ?? false;
   const hasAccessToPreview = config.getHasAccessToPreviewModel?.() ?? false;
+  const useGemini3_5Flash = config.hasGemini35FlashGAAccess?.() ?? false;
 
   // Capture the original family intent before any normalization or early downgrade.
   const isOriginallyGemini3 = isGemini3Model(modelFromConfig, config);
@@ -65,6 +66,7 @@ export function resolvePolicyChain(
       useCustomToolModel,
       hasAccessToPreview,
       config,
+      useGemini3_5Flash,
     ),
   );
   const isAutoPreferred = normalizedPreferredModel
@@ -82,6 +84,7 @@ export function resolvePolicyChain(
     const context = {
       useGemini3_1: useGemini31,
       useCustomTools: useCustomToolModel,
+      useGemini3_5Flash,
     };
 
     if (resolvedModel === DEFAULT_GEMINI_FLASH_LITE_MODEL) {
@@ -136,6 +139,7 @@ export function resolvePolicyChain(
           userTier: config.getUserTier(),
           useGemini31,
           useCustomToolModel,
+          useGemini3_5Flash,
         });
       } else {
         // User requested Gemini 3 but has no access. Proactively downgrade
@@ -146,6 +150,7 @@ export function resolvePolicyChain(
           userTier: config.getUserTier(),
           useGemini31,
           useCustomToolModel,
+          useGemini3_5Flash,
         });
       }
     } else {

packages/core/src/code_assist/experiments/flagNames.ts

@@ -19,6 +19,7 @@ export const ExperimentFlags = {
   GEMINI_3_1_PRO_LAUNCHED: 45760185,
   PRO_MODEL_NO_ACCESS: 45768879,
   DEFAULT_REQUEST_TIMEOUT: 45773134,
+  GEMINI_3_5_FLASH_GA_LAUNCHED: 45780819,
 } as const;
 
 export type ExperimentFlagName =

packages/core/src/config/config.test.ts

@@ -69,6 +69,7 @@ import {
   DEFAULT_GEMINI_MODEL_AUTO,
   PREVIEW_GEMINI_MODEL_AUTO,
   PREVIEW_GEMINI_FLASH_MODEL,
+  DEFAULT_GEMINI_FLASH_MODEL,
 } from './models.js';
 import { Storage } from './storage.js';
 import type { AgentLoopContext } from './agent-loop-context.js';
@@ -4346,3 +4347,34 @@ describe('ADKSettings', () => {
     expect(config.getAgentSessionNoninteractiveEnabled()).toBe(true);
   });
 });
+
+describe('hasGemini35FlashGAAccess model setting', () => {
+  const baseParams: ConfigParameters = {
+    sessionId: 'test',
+    targetDir: '.',
+    debugMode: false,
+    model: 'test-model',
+    cwd: '.',
+  };
+
+  it('should set DEFAULT_GEMINI_FLASH_MODEL and PREVIEW_GEMINI_FLASH_MODEL to gemini-3.5-flash if hasGemini35FlashGAAccess returns true', () => {
+    const config = new Config(baseParams);
+
+    // Set experiment to return true for GEMINI_3_5_FLASH_GA_LAUNCHED
+    config.setExperiments({
+      experimentIds: [],
+      flags: {
+        [ExperimentFlags.GEMINI_3_5_FLASH_GA_LAUNCHED]: {
+          boolValue: true,
+        },
+      },
+    });
+
+    // Call the method
+    const result = config.hasGemini35FlashGAAccess();
+    expect(result).toBe(true);
+
+    expect(DEFAULT_GEMINI_FLASH_MODEL).toBe('gemini-3.5-flash');
+    expect(PREVIEW_GEMINI_FLASH_MODEL).toBe('gemini-3.5-flash');
+  });
+});

packages/core/src/config/config.ts

@@ -86,6 +86,7 @@ import {
   isGemini2Model,
   PREVIEW_GEMINI_FLASH_MODEL,
   resolveModel,
+  setFlashModels,
 } from './models.js';
 import { shouldAttemptBrowserLaunch } from '../utils/browser.js';
 import type { MCPOAuthConfig } from '../mcp/oauth-provider.js';
@@ -2054,6 +2055,7 @@ export class Config implements McpContext, AgentLoopContext {
       this.getUseCustomToolModelSync(),
       this.getHasAccessToPreviewModel(),
       this,
+      this.hasGemini35FlashGAAccess(),
     );
 
     const isPreview = isPreviewModel(primaryModel, this);
@@ -2093,6 +2095,7 @@ export class Config implements McpContext, AgentLoopContext {
       this.getUseCustomToolModelSync(),
       this.getHasAccessToPreviewModel(),
       this,
+      this.hasGemini35FlashGAAccess(),
     );
     return this.modelQuotas.get(primaryModel)?.remaining;
   }
@@ -2108,6 +2111,7 @@ export class Config implements McpContext, AgentLoopContext {
       this.getUseCustomToolModelSync(),
       this.getHasAccessToPreviewModel(),
       this,
+      this.hasGemini35FlashGAAccess(),
     );
     return this.modelQuotas.get(primaryModel)?.limit;
   }
@@ -2123,6 +2127,7 @@ export class Config implements McpContext, AgentLoopContext {
       this.getUseCustomToolModelSync(),
       this.getHasAccessToPreviewModel(),
       this,
+      this.hasGemini35FlashGAAccess(),
     );
     return this.modelQuotas.get(primaryModel)?.resetTime;
   }
@@ -3537,6 +3542,32 @@ export class Config implements McpContext, AgentLoopContext {
     );
   }
 
+  /**
+   * Returns whether Gemini 3.5 Flash GA has been launched.
+   *
+   * Note: This method should only be called after startup, once experiments have been loaded.
+   */
+  hasGemini35FlashGAAccess(): boolean {
+    const authType = this.contentGeneratorConfig?.authType;
+    const hasAccess = (() => {
+      if (this.isGemini31LaunchedForAuthType(authType)) {
+        return true;
+      }
+      return (
+        this.experiments?.flags[ExperimentFlags.GEMINI_3_5_FLASH_GA_LAUNCHED]
+          ?.boolValue ?? false
+      );
+    })();
+    // Used to set default flash models based on access
+    // TODO: Remove once the experiment for 3_5 flash rollut can be cleaned up.
+    if (hasAccess) {
+      setFlashModels('gemini-3.5-flash', 'gemini-3.5-flash');
+    } else {
+      setFlashModels('gemini-3-flash-preview', 'gemini-2.5-flash');
+    }
+    return hasAccess;
+  }
+
   /**
    * Returns whether Gemini 3.1 has been launched.
    *