feat(core): request json response format for locate adapters

Author: EAGzzyCSL

packages/core/src/ai-model/models/glm.ts

@@ -4,6 +4,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from './types';
+import { isLocateIntent } from './utils/intent';
 
 const buildGlmChatCompletionParams = (
   input: ChatCompletionCallContext,
@@ -16,6 +17,12 @@ const buildGlmChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // Zhipu structured output JSON mode:
+  // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const modelSpecificConfig: Record<string, unknown> = {};
 
   if (reasoningEnabled !== 'default') {

packages/core/src/ai-model/models/gpt.ts

@@ -5,11 +5,12 @@ import type {
   ImageDetail,
   ModelAdapterDefinition,
 } from './types';
+import { isLocateIntent } from './utils/intent';
 
 const originalImageDetailForDefaultIntent = (
   input: ChatCompletionCallContext,
 ): ImageDetail | undefined =>
-  input.intent === 'default' || input.requiresOriginalImageDetail
+  isLocateIntent(input.intent) || input.requiresOriginalImageDetail
     ? 'original'
     : undefined;
 
@@ -24,6 +25,12 @@ const buildGpt5ChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // OpenAI Chat Completions JSON mode:
+  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const effectiveReasoningEffort =
     reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';
 

packages/core/src/ai-model/models/kimi.ts

@@ -4,6 +4,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from './types';
+import { isLocateIntent } from './utils/intent';
 
 const buildKimiChatCompletionParams = (
   input: ChatCompletionCallContext,
@@ -16,6 +17,12 @@ const buildKimiChatCompletionParams = (
   // kimi disallow custom temperature
   commonOverrideConfig.temperature = undefined;
 
+  // Kimi Chat Completions response_format:
+  // https://platform.kimi.com/docs/api/chat
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const modelSpecificConfig: Record<string, unknown> = {
     thinking: {
       type: effectiveReasoningEnabled ? 'enabled' : 'disabled',

packages/core/src/ai-model/models/mimo.ts

@@ -4,6 +4,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from './types';
+import { isLocateIntent } from './utils/intent';
 
 const buildMimoChatCompletionParams = (
   input: ChatCompletionCallContext,
@@ -14,9 +15,9 @@ const buildMimoChatCompletionParams = (
 
   // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api
   // Observed with thinking disabled: Mimo needs json_object to return JSON.
-  commonOverrideConfig.response_format = {
-    type: intent === 'default' ? 'json_object' : 'text',
-  };
+  if (isLocateIntent(intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
 
   if (userConfig.temperature !== undefined) {
     commonOverrideConfig.temperature = userConfig.temperature;

packages/core/src/ai-model/models/qwen.ts

@@ -9,6 +9,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from './types';
+import { isLocateIntent } from './utils/intent';
 
 const defaultBboxSize = 20;
 
@@ -65,6 +66,12 @@ const buildQwenChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // Alibaba Cloud Model Studio JSON mode:
+  // https://help.aliyun.com/zh/model-studio/json-mode
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const modelSpecificConfig: Record<string, unknown> = {};
 
   if (reasoningEnabled !== 'default') {

packages/core/src/ai-model/models/utils/intent.ts

@@ -0,0 +1,5 @@
+import type { TIntent } from '@midscene/shared/env';
+
+export function isLocateIntent(intent?: TIntent): boolean {
+  return intent === 'default';
+}

packages/core/tests/unit-test/model-adapter/glm.test.ts

@@ -87,4 +87,17 @@ describe('glm model adapter', () => {
       thinking: { type: 'disabled' },
     });
   });
+
+  it('uses json_object response format for glm-v locate intent', () => {
+    const result = glmAdapter.chatCompletion.buildChatCompletionParams({
+      intent: 'default',
+      userConfig: {},
+    });
+
+    expect(result.config).toEqual({
+      temperature: 0,
+      response_format: { type: 'json_object' },
+      thinking: { type: 'disabled' },
+    });
+  });
 });

packages/core/tests/unit-test/model-adapter/gpt.test.ts

@@ -133,4 +133,13 @@ describe('gpt model adapter', () => {
       reasoning_effort: 'none',
     });
   });
+
+  it('uses json_object response format for gpt-5 locate intent', () => {
+    const result = gpt5Adapter.chatCompletion.buildChatCompletionParams({
+      intent: 'default',
+      userConfig: {},
+    });
+
+    expect(result.config.response_format).toEqual({ type: 'json_object' });
+  });
 });

packages/core/tests/unit-test/model-adapter/kimi.test.ts

@@ -104,4 +104,13 @@ describe('kimi model adapter', () => {
       thinking: { type: 'disabled' },
     });
   });
+
+  it('uses json_object response format for kimi locate intent', () => {
+    const result = kimiAdapter.chatCompletion.buildChatCompletionParams({
+      intent: 'default',
+      userConfig: {},
+    });
+
+    expect(result.config.response_format).toEqual({ type: 'json_object' });
+  });
 });

packages/core/tests/unit-test/model-adapter/mimo.test.ts

@@ -15,7 +15,6 @@ describe('mimo model adapter', () => {
 
     expect(result.config).toEqual({
       temperature: 0,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
   });
@@ -48,7 +47,6 @@ describe('mimo model adapter', () => {
     expect(result.config).toEqual({
       temperature: 0.7,
       seed: 123,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
   });
@@ -69,12 +67,10 @@ describe('mimo model adapter', () => {
 
     expect(disabledResult.config).toEqual({
       temperature: 0,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
     expect(enabledResult.config).toEqual({
       temperature: 0,
-      response_format: { type: 'text' },
       thinking: { type: 'enabled' },
     });
   });
@@ -93,7 +89,6 @@ describe('mimo model adapter', () => {
     ]);
     expect(result.config).toEqual({
       temperature: 0,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
   });
@@ -107,20 +102,18 @@ describe('mimo model adapter', () => {
 
     expect(result.config).toEqual({
       temperature: 0.7,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
   });
 
-  it('uses text response format for planning intent', () => {
+  it('does not set response format for planning intent', () => {
     const result = mimoAdapter.chatCompletion.buildChatCompletionParams({
       intent: 'planning',
       userConfig: {},
     });
 
     expect(result.config).toEqual({
       temperature: 0,
-      response_format: { type: 'text' },
       thinking: { type: 'disabled' },
     });
   });
@@ -138,8 +131,6 @@ describe('mimo model adapter', () => {
     expect(defaultResult.config.response_format).toEqual({
       type: 'json_object',
     });
-    expect(insightResult.config.response_format).toEqual({
-      type: 'text',
-    });
+    expect(insightResult.config.response_format).toBeUndefined();
   });
 });