fix(core): harden aiLocateAll contract

quanru · quanru · commit 0b05ea09bcf8 · 2026-06-23T12:20:29.000+08:00
diff --git a/apps/site/docs/en/api.mdx b/apps/site/docs/en/api.mdx
@@ -1000,15 +1000,15 @@ function aiLocateAll(
       height: number;
     };
     center: [number, number];
-    dpr: number; // device pixel ratio
+    dpr?: number; // device pixel ratio
   }>
 >;
 ```
 
 - Parameters:
 
   - `locate: string | Object` - A natural language description shared by all target elements, or [prompting with images](#prompting-with-images).
-  - `options?: Object` - Optional, a configuration object. `uiContext` and image prompt options are supported. Single-element optimizations such as `xpath`, cache lookup, and `deepLocate` are not applied to `aiLocateAll()`.
+  - `options?: Object` - Optional, a configuration object. `uiContext` and image prompt options are supported. Single-element optimizations such as `xpath`, cache lookup, and `deepLocate` are not applied to `aiLocateAll()`; passing these options throws an error.
 
 - Return Value:
 
diff --git a/apps/site/docs/zh/api.mdx b/apps/site/docs/zh/api.mdx
@@ -993,15 +993,15 @@ function aiLocateAll(
       height: number;
     };
     center: [number, number];
-    dpr: number; // device pixel ratio
+    dpr?: number; // device pixel ratio
   }>
 >;
 ```
 
 - 参数：
 
   - `locate: string | Object` - 所有目标元素共用的自然语言描述，或[使用图片作为提示词](#使用图片作为提示词)。
-  - `options?: Object` - 可选配置对象。支持 `uiContext` 和图片提示词相关配置。`xpath`、定位缓存、`deepLocate` 等单元素定位优化不会应用到 `aiLocateAll()`。
+  - `options?: Object` - 可选配置对象。支持 `uiContext` 和图片提示词相关配置。`xpath`、定位缓存、`deepLocate` 等单元素定位优化不会应用到 `aiLocateAll()`；传入这些选项会抛错。
 
 - 返回值：
 
diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts
@@ -18,6 +18,7 @@ import {
   type ExecutionRecorderItem,
   type ExecutionTask,
   type ExecutionTaskLog,
+  type LocateAllOption,
   type LocateOption,
   type LocateResultElement,
   type OnTaskStartTip,
@@ -96,6 +97,10 @@ const defaultServiceExtractOption: ServiceExtractOption = {
   screenshotIncluded: true,
 };
 
+type LocateAllResultItem = Pick<LocateResultElement, 'rect' | 'center'> & {
+  dpr?: number;
+};
+
 export type AiActOptions = {
   cacheable?: boolean;
   fileChooserAccept?: string | string[];
@@ -111,6 +116,30 @@ type AiActInternalOptions = AiActOptions & {
   };
 };
 
+const unsupportedLocateAllOptionKeys = [
+  'deepLocate',
+  'deepThink',
+  'xpath',
+  'cacheable',
+  'fileChooserAccept',
+] as const;
+
+function assertLocateAllOptionsSupported(opt?: LocateAllOption) {
+  if (!opt || typeof opt !== 'object') {
+    return;
+  }
+
+  const providedUnsupportedKeys = unsupportedLocateAllOptionKeys.filter((key) =>
+    Object.prototype.hasOwnProperty.call(opt, key),
+  );
+  assert(
+    providedUnsupportedKeys.length === 0,
+    `aiLocateAll does not support these single-element locate options: ${providedUnsupportedKeys.join(
+      ', ',
+    )}. Supported options are uiContext and image prompt options.`,
+  );
+}
+
 export class Agent<
   InterfaceType extends AbstractInterface = AbstractInterface,
 > {
@@ -1126,16 +1155,21 @@ export class Agent<
     } as Pick<LocateResultElement, 'rect' | 'center'>;
   }
 
-  async aiLocateAll(prompt: TUserPrompt, opt?: LocateOption) {
+  async aiLocateAll(
+    prompt: TUserPrompt,
+    opt?: LocateAllOption,
+  ): Promise<LocateAllResultItem[]> {
+    assertLocateAllOptionsSupported(opt);
     const locateParam = buildDetailedLocateParam(prompt, opt);
     assert(locateParam, 'cannot get locate param for aiLocateAll');
-    const locatePlan = locatePlanForLocateAll(locateParam);
+    const locateAllParam = { prompt: locateParam.prompt };
+    const locatePlan = locatePlanForLocateAll(locateAllParam);
     const plans = [locatePlan];
     const defaultModel = this.resolveModelRuntime('default');
     const planningModel = this.resolveModelRuntime('planning');
 
     const { output } = await this.taskExecutor.runPlans(
-      taskTitleStr('Locate', locateParamStr(locateParam)),
+      taskTitleStr('LocateAll', locateParamStr(locateAllParam)),
       plans,
       planningModel,
       defaultModel,
diff --git a/packages/core/src/agent/ui-utils.ts b/packages/core/src/agent/ui-utils.ts
@@ -150,6 +150,7 @@ export type TaskTitleType =
   | 'Assert'
   | 'WaitFor'
   | 'Locate'
+  | 'LocateAll'
   | 'Markdown'
   | 'Boolean'
   | 'Number'
diff --git a/packages/core/src/ai-model/inspect.ts b/packages/core/src/ai-model/inspect.ts
@@ -515,9 +515,24 @@ export async function AiLocateAllElements(
   const errors: string[] = Array.isArray(res.content.errors)
     ? [...res.content.errors]
     : [];
-  const rawElements = Array.isArray(res.content.elements)
-    ? res.content.elements
-    : [];
+  if (!Array.isArray(res.content.elements)) {
+    return {
+      rawResponse,
+      rawChoiceMessage: res.rawChoiceMessage,
+      usage: res.usage,
+      reasoning_content: res.reasoning_content,
+      parseResult: {
+        elements: [],
+        errors: [
+          ...errors,
+          'AI response error: locate all response must contain an elements array',
+        ],
+        fatalError: true,
+      },
+    };
+  }
+
+  const rawElements = res.content.elements;
   const elements: LocateResultElement[] = [];
 
   rawElements.forEach((rawElement, index) => {
@@ -543,6 +558,11 @@ export async function AiLocateAllElements(
     }
   });
 
+  const fatalError = rawElements.length > 0 && elements.length === 0;
+  if (fatalError) {
+    errors.push('AI response error: failed to parse every locate all element');
+  }
+
   return {
     rawResponse,
     rawChoiceMessage: res.rawChoiceMessage,
@@ -551,6 +571,7 @@ export async function AiLocateAllElements(
     parseResult: {
       elements: sortAndDedupeLocateElements(elements),
       errors,
+      fatalError,
     },
   };
 }
diff --git a/packages/core/src/yaml.ts b/packages/core/src/yaml.ts
@@ -18,6 +18,11 @@ export interface LocateOption extends Partial<TMultimodalPrompt> {
   fileChooserAccept?: string | string[]; // file path(s) to upload when tapping triggers a file chooser
 }
 
+export interface LocateAllOption extends Partial<TMultimodalPrompt> {
+  prompt?: TUserPrompt;
+  uiContext?: UIContext;
+}
+
 export interface ServiceExtractOption {
   domIncluded?: boolean | 'visible-only';
   screenshotIncluded?: boolean;
diff --git a/packages/core/tests/unit-test/agent-locate-all.test.ts b/packages/core/tests/unit-test/agent-locate-all.test.ts
@@ -48,20 +48,15 @@ describe('Agent.aiLocateAll', () => {
         runner: {} as any,
       });
 
-    const result = await agent.aiLocateAll('all submit buttons', {
-      cacheable: false,
-    });
+    const result = await agent.aiLocateAll('all submit buttons');
 
     expect(runPlans).toHaveBeenCalledWith(
-      'Locate - all submit buttons',
+      'LocateAll - all submit buttons',
       [
         {
           type: 'LocateAll',
           param: {
             prompt: 'all submit buttons',
-            cacheable: false,
-            deepLocate: false,
-            xpath: undefined,
           },
           thought: '',
         },
@@ -82,4 +77,31 @@ describe('Agent.aiLocateAll', () => {
       },
     ]);
   });
+
+  it('rejects single-element locate options that are unsupported by aiLocateAll', async () => {
+    const agent = new Agent(createMockInterface(), {
+      generateReport: false,
+      modelConfig,
+    });
+    const runPlans = vi.spyOn(agent.taskExecutor, 'runPlans');
+
+    await expect(
+      agent.aiLocateAll('all submit buttons', { cacheable: false } as any),
+    ).rejects.toThrow(
+      /aiLocateAll does not support these single-element locate options: cacheable/,
+    );
+    await expect(
+      agent.aiLocateAll('all submit buttons', { deepLocate: true } as any),
+    ).rejects.toThrow(
+      /aiLocateAll does not support these single-element locate options: deepLocate/,
+    );
+    await expect(
+      agent.aiLocateAll('all submit buttons', {
+        xpath: '//*[@id="submit"]',
+      } as any),
+    ).rejects.toThrow(
+      /aiLocateAll does not support these single-element locate options: xpath/,
+    );
+    expect(runPlans).not.toHaveBeenCalled();
+  });
 });
diff --git a/packages/core/tests/unit-test/inspect-locate-all.test.ts b/packages/core/tests/unit-test/inspect-locate-all.test.ts
@@ -64,4 +64,51 @@ describe('AiLocateAllElements', () => {
     expect(result.usage).toEqual({ total_tokens: 10 });
     expect(result.reasoning_content).toBe('all locate reasoning');
   });
+
+  it('marks malformed top-level locate-all responses as fatal', async () => {
+    vi.mocked(callAIWithObjectResponse).mockResolvedValue({
+      content: {
+        bbox: [100, 100, 120, 120],
+      },
+      rawChoiceMessage: { role: 'assistant' },
+      contentString: '{}',
+    });
+
+    const result = await AiLocateAllElements({
+      context: createFakeContext(),
+      targetElementDescription: 'submit buttons',
+      modelRuntime: getModelRuntime(modelConfig),
+    });
+
+    expect(result.parseResult).toEqual({
+      elements: [],
+      errors: [
+        'AI response error: locate all response must contain an elements array',
+      ],
+      fatalError: true,
+    });
+  });
+
+  it('marks responses as fatal when every returned candidate fails to parse', async () => {
+    vi.mocked(callAIWithObjectResponse).mockResolvedValue({
+      content: {
+        elements: [{ bbox: [10, 20, Number.NaN, 40] }],
+      },
+      rawChoiceMessage: { role: 'assistant' },
+      contentString: '{}',
+    });
+
+    const result = await AiLocateAllElements({
+      context: createFakeContext(),
+      targetElementDescription: 'submit buttons',
+      modelRuntime: getModelRuntime(modelConfig),
+    });
+
+    expect(result.parseResult.elements).toEqual([]);
+    expect(result.parseResult.errors).toEqual([
+      expect.stringMatching(/Failed to parse locate result #1/),
+      'AI response error: failed to parse every locate all element',
+    ]);
+    expect(result.parseResult.fatalError).toBe(true);
+  });
 });
diff --git a/packages/core/tests/unit-test/player-action-dispatch.test.ts b/packages/core/tests/unit-test/player-action-dispatch.test.ts
@@ -413,7 +413,6 @@ describe('player action dispatch ordering', () => {
         flow: [
           {
             aiLocateAll: 'all delete buttons',
-            cacheable: false,
             name: 'buttons',
           },
         ],
@@ -424,9 +423,7 @@ describe('player action dispatch ordering', () => {
 
       await player.playTask(taskStatus, agent);
 
-      expect(agent.aiLocateAll).toHaveBeenCalledWith('all delete buttons', {
-        cacheable: false,
-      });
+      expect(agent.aiLocateAll).toHaveBeenCalledWith('all delete buttons', {});
       expect(player.result.buttons).toBe(locateAllResult);
     });
 
