test: update all the eval scenarios to work with experimentalPageIdRouting flag (#2085)

Refs #2052 

This change updates all the eval scenarios so that they work with and
without experimentalPageIdRouting flag

Co-authored-by: Samiya Caur <samiyac@chromium.org>

Author: samiyac

scripts/eval_gemini.ts

@@ -19,23 +19,10 @@ const ROOT_DIR = path.resolve(import.meta.dirname, '..');
 const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
 const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');
 
-// Define schema for our test scenarios
-export interface CapturedFunctionCall {
-  name: string;
-  args: Record<string, unknown>;
-}
-
-export interface TestScenario {
-  prompt: string;
-  maxTurns: number;
-  expectations: (calls: CapturedFunctionCall[]) => void;
-  htmlRoute?: {
-    path: string;
-    htmlContent: string;
-  };
-  /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
-  serverArgs?: string[];
-}
+import type {CapturedFunctionCall, TestScenario} from './eval_result.ts';
+import {Result} from './eval_result.ts';
+export type {CapturedFunctionCall, TestScenario};
+export {Result};
 
 async function loadScenario(scenarioPath: string): Promise<TestScenario> {
   const module = await import(pathToFileURL(scenarioPath).href);
@@ -54,6 +41,7 @@ async function runSingleScenario(
   modelId: string,
   debug: boolean,
   includeSkill: boolean,
+  extraServerArgs: string[] = [],
 ): Promise<void> {
   const debugLog = (...args: unknown[]) => {
     if (debug) {
@@ -125,6 +113,9 @@ async function runSingleScenario(
     if (scenario.serverArgs) {
       args.push(...scenario.serverArgs);
     }
+    if (extraServerArgs.length > 0) {
+      args.push(...extraServerArgs);
+    }
 
     transport = new StdioClientTransport({
       command: 'node',
@@ -175,7 +166,7 @@ async function runSingleScenario(
     debugLog(`\n--- Response ---\n${result.text}`);
 
     debugLog('\nVerifying expectations...');
-    scenario.expectations(allCalls);
+    scenario.expectations(new Result(allCalls, args));
   } finally {
     try {
       await client?.close();
@@ -195,7 +186,7 @@ async function main() {
     options: {
       model: {
         type: 'string',
-        default: 'gemini-2.5-flash',
+        default: 'gemini-3-flash-preview',
       },
       debug: {
         type: 'boolean',
@@ -209,6 +200,9 @@ async function main() {
         type: 'boolean',
         default: false,
       },
+      'server-args': {
+        type: 'string',
+      },
     },
     allowPositionals: true,
   });
@@ -217,6 +211,9 @@ async function main() {
   const debug = values.debug;
   const repeat = values.repeat;
   const includeSkill = values['include-skill'];
+  const extraServerArgs = values['server-args']
+    ? values['server-args'].split(/\s+/)
+    : [];
 
   const scenarioFiles =
     positionals.length > 0
@@ -248,6 +245,7 @@ async function main() {
             modelId,
             debug,
             includeSkill,
+            extraServerArgs,
           );
           console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);
           successCount++;

scripts/eval_result.ts

@@ -0,0 +1,107 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'node:assert';
+
+export interface CapturedFunctionCall {
+  name: string;
+  args: Record<string, unknown>;
+}
+
+export class Result {
+  private nextCallIndex = 0;
+  public readonly calls: CapturedFunctionCall[];
+  public readonly serverArgs: string[];
+
+  constructor(calls: CapturedFunctionCall[], serverArgs: string[]) {
+    this.calls = calls;
+    this.serverArgs = serverArgs;
+  }
+
+  get hasPageIdRouting(): boolean {
+    return this.serverArgs.includes('--experimental-page-id-routing');
+  }
+
+  get remainingCalls(): CapturedFunctionCall[] {
+    return this.calls.slice(this.nextCallIndex);
+  }
+
+  /**
+   * Consumes initial page navigation/setup boilerplate.
+   * - Ignores/skips leading list_pages calls.
+   * - Asserts that new_page or navigate_page was called.
+   * - Determines the expected pageId.
+   * - Returns the active pageId.
+   */
+  consumePageNavigation(): number | undefined {
+    if (this.calls[this.nextCallIndex]?.name === 'list_pages') {
+      this.nextCallIndex++;
+    }
+
+    const navCall = this.calls[this.nextCallIndex];
+    assert.ok(
+      navCall &&
+        (navCall.name === 'new_page' || navCall.name === 'navigate_page'),
+      `Expected navigation call (new_page or navigate_page), but got: ${navCall?.name || 'none'}`,
+    );
+    this.nextCallIndex++;
+
+    const isNewPage = navCall.name === 'new_page';
+    let pageId: number | undefined;
+    if (this.hasPageIdRouting) {
+      pageId = isNewPage ? 2 : 1;
+    }
+
+    return pageId;
+  }
+
+  /**
+   * Asserts that the next call in sequence has the correct name and matches expected arguments.
+   * Increments the internal call index.
+   */
+  assertNextCall(
+    name: string,
+    expectedArgs?: Record<string, unknown>,
+  ): CapturedFunctionCall {
+    const call = this.calls[this.nextCallIndex];
+    assert.ok(
+      call,
+      `Expected call at index ${this.nextCallIndex} (name: '${name}') to exist`,
+    );
+    assert.strictEqual(
+      call.name,
+      name,
+      `Expected call at index ${this.nextCallIndex} to be '${name}', but got '${call.name}'`,
+    );
+
+    if (expectedArgs) {
+      for (const entry of Object.entries(expectedArgs)) {
+        const key = entry[0];
+        const value = entry[1];
+        assert.deepStrictEqual(
+          call.args[key],
+          value,
+          `Expected argument '${key}' on call '${name}' to be ${JSON.stringify(value)}, got ${JSON.stringify(call.args[key])}`,
+        );
+      }
+    }
+
+    this.nextCallIndex++;
+    return call;
+  }
+}
+
+export interface TestScenario {
+  prompt: string;
+  maxTurns: number;
+  expectations: (result: Result) => void;
+  htmlRoute?: {
+    path: string;
+    htmlContent: string;
+  };
+  /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
+  serverArgs?: string[];
+}

scripts/eval_scenarios/console_test.ts

@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';
 
 export const scenario: TestScenario = {
   prompt: 'Navigate to <TEST_URL> and check the console messages.',
-  maxTurns: 2,
+  maxTurns: 3,
   htmlRoute: {
     path: '/console_test.html',
     htmlContent: `
@@ -20,16 +20,12 @@ export const scenario: TestScenario = {
       </script>
     `,
   },
-  expectations: calls => {
-    assert.strictEqual(calls.length, 2);
-    assert.ok(
-      calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
-      'First call should be navigation',
-    );
-    assert.strictEqual(
-      calls[1].name,
+  expectations: result => {
+    const pageId = result.consumePageNavigation();
+    assert.strictEqual(result.remainingCalls.length, 1);
+    result.assertNextCall(
       'list_console_messages',
-      'Second call should be list_console_messages',
+      result.hasPageIdRouting ? {pageId} : undefined,
     );
   },
 };

scripts/eval_scenarios/emulation_test.ts

@@ -11,9 +11,12 @@ import type {TestScenario} from '../eval_gemini.ts';
 export const scenario: TestScenario = {
   prompt: 'Emulate offline network conditions.',
   maxTurns: 2,
-  expectations: calls => {
-    assert.strictEqual(calls.length, 1);
-    assert.strictEqual(calls[0].name, 'emulate');
-    assert.strictEqual(calls[0].args.networkConditions, 'Offline');
+  expectations: result => {
+    assert.ok(result.remainingCalls.length >= 1);
+    result.assertNextCall('list_pages');
+    result.assertNextCall('emulate', {
+      networkConditions: 'Offline',
+      pageId: result.hasPageIdRouting ? 1 : undefined,
+    });
   },
 };

scripts/eval_scenarios/emulation_userAgent_test.ts

@@ -9,14 +9,20 @@ import assert from 'node:assert';
 import type {TestScenario} from '../eval_gemini.ts';
 
 export const scenario: TestScenario = {
-  prompt: 'Emulate iPhone 14 user agent',
+  prompt: 'Emulate current page with iPhone 14 user agent',
   maxTurns: 2,
-  expectations: calls => {
-    assert.strictEqual(calls.length, 1);
-    assert.strictEqual(calls[0].name, 'emulate');
-    assert.deepStrictEqual(
-      calls[0].args.userAgent,
-      'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
-    );
+  expectations: result => {
+    assert.ok(result.remainingCalls.length >= 1);
+    if (
+      result.hasPageIdRouting ||
+      result.remainingCalls[0]?.name === 'list_pages'
+    ) {
+      result.assertNextCall('list_pages');
+    }
+    result.assertNextCall('emulate', {
+      userAgent:
+        'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
+      pageId: result.hasPageIdRouting ? 1 : undefined,
+    });
   },
 };

scripts/eval_scenarios/emulation_viewport_test.ts

@@ -9,11 +9,19 @@ import assert from 'node:assert';
 import type {TestScenario} from '../eval_gemini.ts';
 
 export const scenario: TestScenario = {
-  prompt: 'Emulate iPhone 14 viewport',
+  prompt: 'Emulate current page with iPhone 14 viewport',
   maxTurns: 2,
-  expectations: calls => {
-    assert.strictEqual(calls.length, 1);
-    assert.strictEqual(calls[0].name, 'emulate');
-    assert.deepStrictEqual(calls[0].args.viewport, '390x844x3,mobile,touch');
+  expectations: result => {
+    assert.ok(result.remainingCalls.length >= 1);
+    if (
+      result.hasPageIdRouting ||
+      result.remainingCalls[0]?.name === 'list_pages'
+    ) {
+      result.assertNextCall('list_pages');
+    }
+    result.assertNextCall('emulate', {
+      viewport: '390x844x3,mobile,touch',
+      pageId: result.hasPageIdRouting ? 1 : undefined,
+    });
   },
 };

scripts/eval_scenarios/fill_select_and_checkboxes_test.ts

@@ -11,7 +11,7 @@ import type {TestScenario} from '../eval_gemini.ts';
 export const scenario: TestScenario = {
   prompt:
     'Go to <TEST_URL>, fill the form with size = 2 CPUs and components = [docker, nginx].',
-  maxTurns: 4, // allow for at least one extra turn to verify there are no extra clicks after fill_form
+  maxTurns: 5, // allow for at least one extra turn to verify there are no extra clicks after fill_form
   htmlRoute: {
     path: '/input_test.html',
     htmlContent: `
@@ -40,32 +40,42 @@ export const scenario: TestScenario = {
       </form>
     `,
   },
-  expectations: calls => {
-    assert.ok(calls.length >= 3, 'Not enough calls made');
-    assert.ok(
-      calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+  expectations: result => {
+    const pageId = result.consumePageNavigation();
+    assert.ok(result.remainingCalls.length >= 2, 'Not enough calls made');
+    result.assertNextCall(
+      'take_snapshot',
+      result.hasPageIdRouting ? {pageId} : undefined,
+    );
+    const fillFormCall = result.assertNextCall(
+      'fill_form',
+      result.hasPageIdRouting ? {pageId} : undefined,
     );
-    assert.strictEqual(calls[1].name, 'take_snapshot');
-    assert.strictEqual(calls[2].name, 'fill_form');
 
-    const elements = calls[2].args.elements as Array<{
-      uid: string;
-      value: string;
-    }>;
+    const elements = fillFormCall.args.elements;
+    assert.ok(Array.isArray(elements), 'elements should be an array');
     assert.strictEqual(
       elements.length,
       3,
       'fill_form should be used with all form elements at once',
     );
 
-    const uids = new Set(elements.map(e => e.uid));
+    const typedElements = elements.map(e => {
+      assert.ok(e && typeof e === 'object' && 'uid' in e && 'value' in e);
+      return {
+        uid: String(e.uid),
+        value: String(e.value),
+      };
+    });
+
+    const uids = new Set(typedElements.map(e => e.uid));
     assert.strictEqual(
       uids.size,
       3,
       'fill_form should target three distinct elements',
     );
 
-    const values = elements.map(e => e.value).sort();
+    const values = typedElements.map(e => e.value).sort();
     assert.deepStrictEqual(
       values,
       ['2 vCPU, 4GB RAM', 'true', 'true'],
@@ -74,11 +84,9 @@ export const scenario: TestScenario = {
 
     const submitUid = '1_15';
 
-    const extraFormInteractions = calls
-      .slice(3)
-      .filter(
-        c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,
-      );
+    const extraFormInteractions = result.remainingCalls.filter(
+      c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,
+    );
     assert.deepEqual(
       extraFormInteractions.length,
       0,