Skip to content

Commit bf2098d

Browse files
dxcsmamdxcsmam
authored
Merge pull request #4 from Core-Mate/harvey/configure-vlm-env
Configure graph agents with VLM env
2 parents 154af79 + ac7eafe commit bf2098d

7 files changed

Lines changed: 105 additions & 103 deletions

File tree

‎client/start.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ info "Install completed"
5757
# --------------------------------------------------
5858
# 5. Launch app
5959
# --------------------------------------------------
60-
PACKAGE="com.haomai.promotor"
60+
PACKAGE="com.coremate.opengui"
6161
adb shell am start -n "$PACKAGE/.login.SplashActivity" >/dev/null 2>&1
6262
info "App launched"
6363

‎server/apps/backend/.env.example‎

Lines changed: 5 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,5 @@
1-
# OpenGUI Server environment configuration
2-
#
3-
# Quick start:
4-
# 1. Copy this file to .env
5-
# cp apps/backend/.env.example apps/backend/.env
6-
# 2. Fill in the required model settings below.
7-
# 3. Never commit the real .env file or real API keys.
1+
# OpenGUI Server Environment Configuration
2+
# Copy this file to .env and fill in your values
83

94
# App
105
NODE_ENV=development
@@ -19,42 +14,10 @@ REDIS_PORT=6379
1914
REDIS_DB=0
2015
REDIS_PASSWORD=
2116

22-
# ============================================================
23-
# Required model configuration
24-
# ============================================================
25-
#
26-
# OpenGUI uses these variables at runtime:
27-
# - CLAUDE_* is the default text/action model config for:
28-
# plan-supervisor, summarizer, executor-a11y, and action-summarizer.
29-
# - VLM_* is the vision model config for executor-vlm.
30-
#
31-
# CLAUDE_BASE_URL and VLM_BASE_URL are optional when you use the provider's
32-
# default endpoint. Set them when using an OpenAI-compatible gateway.
33-
#
34-
# OpenAI-compatible example:
35-
# CLAUDE_BASE_URL=https://your-openai-compatible-endpoint/v1
36-
# CLAUDE_MODEL=your-text-model
37-
# CLAUDE_SMALL_MODEL=your-small-or-cheaper-text-model
38-
# VLM_BASE_URL=https://your-openai-compatible-endpoint/v1
39-
# VLM_MODEL=your-vision-model
40-
#
41-
# Do not put real API keys in this example file.
42-
43-
# Text/action agents
44-
CLAUDE_API_KEY=
45-
CLAUDE_BASE_URL=
46-
CLAUDE_MODEL=
47-
CLAUDE_SMALL_MODEL=
48-
49-
# Vision agent
17+
# AI model config (used by all graph agents)
5018
VLM_API_KEY=
51-
VLM_BASE_URL=
52-
VLM_MODEL=
53-
54-
# Optional: Creator Agent (Claude Agent SDK)
55-
# Only required if you use creator-agent SDK features.
56-
ANTHROPIC_API_KEY=
57-
# ANTHROPIC_BASE_URL=
19+
VLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
20+
VLM_MODEL=qwen3.6-plus
5821

5922
# LangSmith tracing (optional, for debugging agent execution)
6023
# LANGSMITH_TRACING=true

‎server/apps/backend/src/modules/graph-agent/config/agent-config.provider.ts‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -45,50 +45,39 @@ export class AgentConfigProvider {
4545

4646
/**
4747
*
48-
* - executor-vlm: VLM_API_KEY, VLM_BASE_URL, VLM_MODEL
49-
* - action-summarizer: CLAUDE_API_KEY, CLAUDE_BASE_URL, CLAUDE_SMALL_MODEL
50-
* - Other Agent: CLAUDE_API_KEY, CLAUDE_BASE_URL, CLAUDE_MODEL
48+
* All graph agents use the same OpenAI-compatible model config:
49+
* - VLM_API_KEY
50+
* - VLM_BASE_URL
51+
* - VLM_MODEL
5152
*
5253
*/
5354
async getModelConfig(agentName: AgentName, region = "CN"): Promise<ModelConfig> {
5455
const config = await this.getConfig(agentName, region);
5556

56-
const isVLM = agentName === AgentName.EXECUTOR_VLM;
57-
const isSmall = agentName === AgentName.ACTION_SUMMARIZER;
57+
const apiKey = this.configService.get<string>("VLM_API_KEY");
58+
const baseURL = this.configService.get<string>("VLM_BASE_URL");
59+
const model = this.configService.get<string>("VLM_MODEL");
5860

59-
const apiKey = isVLM
60-
? this.configService.get<string>("VLM_API_KEY")
61-
: this.configService.get<string>("CLAUDE_API_KEY");
62-
const baseURL = isVLM
63-
? this.configService.get<string>("VLM_BASE_URL")
64-
: this.configService.get<string>("CLAUDE_BASE_URL");
65-
const envModel = isVLM
66-
? this.configService.get<string>("VLM_MODEL")
67-
: isSmall
68-
? this.configService.get<string>("CLAUDE_SMALL_MODEL")
69-
: this.configService.get<string>("CLAUDE_MODEL");
70-
71-
const model = envModel;
7261

7362
if (!apiKey) {
7463
throw new Error(
7564
`API key not configured for agent: ${agentName}. ` +
76-
`Set ${isVLM ? "VLM_API_KEY" : "CLAUDE_API_KEY"} in .env`,
65+
"Set VLM_API_KEY in .env",
7766
);
7867
}
7968

8069
if (!model) {
8170
throw new Error(
8271
`Model not configured for agent: ${agentName}. ` +
83-
`Set ${isVLM ? "VLM_MODEL" : "CLAUDE_MODEL"} in .env`,
72+
"Set VLM_MODEL in .env",
8473
);
8574
}
8675

8776
return {
8877
model,
8978
apiKey,
9079
baseURL: baseURL || undefined,
91-
fallbackModel: config.fallbackModel ?? undefined,
80+
fallbackModel: model,
9281
temperature: config.temperature ?? undefined,
9382
maxTokens: config.maxTokens ?? undefined,
9483
topP: config.topP ?? undefined,

‎server/apps/backend/src/modules/graph-agent/config/chat-model.factory.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { ChatOpenAI } from "@langchain/openai";
2+
import type { ModelConfig } from "./types";
3+
4+
/**
5+
* Creates the single OpenAI-compatible chat model used by OpenGUI graph agents.
6+
*
7+
* Provider credentials and model selection come from AgentConfigProvider, which
8+
* is backed by VLM_API_KEY, VLM_BASE_URL, and VLM_MODEL.
9+
*/
10+
export function createConfiguredChatModel(
11+
config: ModelConfig,
12+
options: {
13+
model?: string;
14+
temperature?: number;
15+
maxTokens?: number;
16+
maxRetries?: number;
17+
timeout?: number;
18+
topP?: number;
19+
} = {},
20+
) {
21+
return new ChatOpenAI({
22+
model: options.model ?? config.model,
23+
apiKey: config.apiKey,
24+
temperature: options.temperature ?? config.temperature,
25+
maxTokens: options.maxTokens ?? config.maxTokens,
26+
maxRetries: options.maxRetries,
27+
timeout: options.timeout,
28+
topP: options.topP ?? config.topP,
29+
...(config.baseURL && {
30+
configuration: { baseURL: config.baseURL },
31+
}),
32+
});
33+
}

‎server/apps/backend/src/modules/graph-agent/graph/nodes/plan-supervisor.node.spec.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
/**
22
* Plan Supervisor 结构化输出集成测试
33
*
4-
* 验证 ChatAnthropic + createAgent + providerStrategy 能否拿到 structuredResponse
4+
* 验证 ChatOpenAI + createAgent + providerStrategy 能否拿到 structuredResponse
55
* 使用真实模型调用，mock 数据尽量简化
66
*
77
* 运行：pnpm test -- plan-supervisor.node.spec
88
*/
99

10-
import { ChatAnthropic } from "@langchain/anthropic";
1110
import { HumanMessage } from "@langchain/core/messages";
1211
import { tool } from "@langchain/core/tools";
12+
import { ChatOpenAI } from "@langchain/openai";
1313
import { createAgent, providerStrategy } from "langchain";
1414
import { z } from "zod";
1515

1616
// ====== API 配置（与 plan-supervisor.node.ts 保持一致）======
17-
const API_KEY = process.env.CLAUDE_API_KEY ?? "test-api-key-placeholder";
18-
const BASE_URL = process.env.CLAUDE_BASE_URL ?? "https://ai-gateway.vercel.sh";
19-
const MODEL = process.env.CLAUDE_MODEL ?? "anthropic/claude-sonnet-4.6";
17+
const API_KEY = process.env.VLM_API_KEY ?? "test-api-key-placeholder";
18+
const BASE_URL =
19+
process.env.VLM_BASE_URL ?? "https://dashscope.aliyuncs.com/compatible-mode/v1";
20+
const MODEL = process.env.VLM_MODEL ?? "qwen3.6-plus";
2021
// =============================================================
2122

2223
const SupervisorOutputSchema = z.object({
@@ -72,18 +73,17 @@ const mockLoadSkill = tool(
7273
},
7374
);
7475

75-
describe("PlanSupervisor - ChatAnthropic 结构化输出集成测试", () => {
76-
let model: ChatAnthropic;
76+
describe("PlanSupervisor - ChatOpenAI 结构化输出集成测试", () => {
77+
let model: ChatOpenAI;
7778

7879
beforeAll(() => {
79-
model = new ChatAnthropic({
80+
model = new ChatOpenAI({
8081
model: MODEL,
8182
apiKey: API_KEY,
82-
clientOptions: {
83+
maxRetries: 2,
84+
timeout: 120000,
85+
configuration: {
8386
baseURL: BASE_URL,
84-
maxRetries: 2,
85-
timeout: 120000,
86-
authToken: null,
8787
},
8888
});
8989
});

‎server/apps/backend/src/modules/graph-agent/graph/utils/execution-connection.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
const CONNECTION_LOST_PATTERNS = [
2+
"no connection for execution",
3+
"socket disconnected",
4+
"client disconnected",
5+
"connection lost",
6+
"ack timeout",
7+
"operation has timed out",
8+
"timeout",
9+
];
10+
11+
function getErrorText(error: unknown): string {
12+
if (error instanceof Error) {
13+
return `${error.name} ${error.message}`.toLowerCase();
14+
}
15+
if (typeof error === "string") {
16+
return error.toLowerCase();
17+
}
18+
return "";
19+
}
20+
21+
export function isExecutionConnectionLost(error: unknown): boolean {
22+
return isExecutionConnectionLostMessage(getErrorText(error));
23+
}
24+
25+
export function isExecutionConnectionLostMessage(message: unknown): boolean {
26+
if (typeof message !== "string") return false;
27+
const lower = message.toLowerCase();
28+
return CONNECTION_LOST_PATTERNS.some((pattern) => lower.includes(pattern));
29+
}
30+
31+
export function buildExecutionConnectionLostMessage(executionId: number): string {
32+
return `Execution ${executionId} lost the client connection. Reconnect the device and retry.`;
33+
}

‎server/start.sh‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,9 @@ ENV_EXAMPLE=apps/backend/.env.example
8383
if [ ! -f "$ENV_FILE" ]; then
8484
if [ -f "$ENV_EXAMPLE" ]; then
8585
cp "$ENV_EXAMPLE" "$ENV_FILE"
86-
warn ".env was created from .env.example. Please edit it and add the required model configuration:"
86+
warn ".env was created from .env.example. Please edit it and add your API keys:"
8787
warn " File: $ENV_FILE"
88-
warn " Required: CLAUDE_API_KEY, CLAUDE_MODEL, CLAUDE_SMALL_MODEL, VLM_API_KEY, VLM_MODEL"
89-
warn " Optional: CLAUDE_BASE_URL, VLM_BASE_URL for OpenAI-compatible gateways"
90-
warn " Optional: ANTHROPIC_API_KEY only for Creator Agent SDK features"
88+
warn " VLM_API_KEY, VLM_BASE_URL, VLM_MODEL"
9189
warn "Run this script again after editing the file."
9290
exit 0
9391
else
@@ -97,30 +95,16 @@ fi
9795

9896
set -a; source "$ENV_FILE" 2>/dev/null || true; set +a
9997

100-
REQUIRED_MODEL_VARS=(
101-
CLAUDE_API_KEY
102-
CLAUDE_MODEL
103-
CLAUDE_SMALL_MODEL
104-
VLM_API_KEY
105-
VLM_MODEL
106-
)
107-
108-
MISSING_MODEL_VARS=()
109-
for var_name in "${REQUIRED_MODEL_VARS[@]}"; do
110-
if [ -z "${!var_name}" ]; then
111-
MISSING_MODEL_VARS+=("$var_name")
112-
fi
113-
done
98+
if [ -z "$VLM_API_KEY" ]; then
99+
warn "VLM_API_KEY is not set. Please edit .env."
100+
fi
114101

115-
if [ "${#MISSING_MODEL_VARS[@]}" -gt 0 ]; then
116-
warn "Missing required model configuration in $ENV_FILE:"
117-
for var_name in "${MISSING_MODEL_VARS[@]}"; do
118-
warn " $var_name"
119-
done
120-
warn "Fill these values and run ./start.sh again."
121-
warn "If text and vision use the same provider, fill both CLAUDE_* and VLM_* explicitly."
122-
warn "ANTHROPIC_API_KEY is optional unless you use Creator Agent SDK features."
123-
exit 1
102+
if [ -z "$VLM_MODEL" ]; then
103+
warn "VLM_MODEL is not set. Please edit .env."
104+
fi
105+
106+
if [ -z "$VLM_BASE_URL" ]; then
107+
warn "VLM_BASE_URL is not set. Please edit .env."
124108
fi
125109

126110
info ".env loaded"

0 commit comments

Comments
 (0)