Skip to content

Commit 7f69ea2

Browse files
committed
feat(chat): upgrade default models, refactor tool system and fix CyberNews integration
- Update LLM default models: OpenAI gpt-4o → gpt-5.3-chat-latest, Anthropic claude-sonnet-4 → claude-opus-4-6 - Update image gen default model: OpenAI dall-e-3 → gpt-image-1.5 - Refactor app tool system from per-app tool definitions to generic app_action + list_apps tools for better scalability - Simplify system prompt and default to English response language - Fix CyberNews app_id mismatch (10008 → 14) - Clean up image gen provider labels and remove nano-banana-2 hint - Replace favicon with custom OpenRoom icon
1 parent b44983b commit 7f69ea2

9 files changed

Lines changed: 179 additions & 163 deletions

File tree

apps/webuiapps/index.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
66
<meta name="format-detection" content="telephone=no" />
77
<meta http-equiv="Content-Securiy-Policy" content="script-src 'self'" />
8-
<link rel="icon" type="image/png" href="/favicon.png" />
8+
<link rel="icon" type="image/jpeg" href="/favicon.jpg" />
99
<title>OpenRoom</title>
1010
<style>
1111
#root,

apps/webuiapps/public/favicon.jpg

45.7 KB
Loading

apps/webuiapps/src/components/ChatPanel/index.tsx

Lines changed: 108 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@ import {
1717
type ImageGenProvider,
1818
} from '@/lib/imageGenClient';
1919
import {
20-
getToolDefinitions,
21-
parseToolName,
20+
getAppActionToolDefinition,
21+
resolveAppAction,
22+
getListAppsToolDefinition,
23+
executeListApps,
2224
APP_REGISTRY,
2325
loadActionsFromMeta,
2426
} from '@/lib/appRegistry';
27+
import { seedMetaFiles } from '@/lib/seedMeta';
2528
import { dispatchAgentAction, onUserAction } from '@/lib/vibeContainerMock';
2629
import { getFileToolDefinitions, isFileTool, executeFileTool } from '@/lib/fileTools';
2730
import {
@@ -38,58 +41,23 @@ interface DisplayMessage {
3841
imageUrl?: string;
3942
}
4043

41-
function buildSystemPrompt(): string {
42-
const appRows = APP_REGISTRY.filter((a) => a.appName !== 'os')
43-
.map((a) => `| ${a.displayName} | ${a.appName} |`)
44-
.join('\n');
45-
46-
return `You are a helpful assistant that can interact with various apps on the user's device.
47-
You have two types of tools:
48-
49-
## App Action Tools
50-
Tool names formatted as "{appName}__{ACTION_TYPE}". Use these to trigger app actions (e.g. refresh UI, play music).
51-
All parameters are strings.
52-
53-
## File Tools
54-
- file_list: List files in a directory. Path relative to workspace root.
55-
- file_read: Read a file's content. Path relative to workspace root.
56-
- file_write: Write content to a file. Path relative to workspace root.
57-
- file_delete: Delete a file from storage. Path relative to workspace root.
58-
59-
App data is stored at "apps/{appName}/data/". Each app also has "apps/{appName}/meta.yaml" (capabilities) and "apps/{appName}/guide.md" (data schema).
60-
61-
App name mapping (use these exact names in file paths and tool prefixes):
62-
| Display Name | appName |
63-
|---|---|
64-
${appRows}
65-
66-
## Workflow
67-
Use file tools to read/write app data, then use the app action tool to notify the app to reload.
68-
69-
### Steps:
70-
1. file_read("apps/{appName}/meta.yaml") to understand the app's capabilities and actions
71-
2. file_read("apps/{appName}/guide.md") to learn the data structure and JSON schema
72-
3. file_list / file_read to explore existing data in "apps/{appName}/data/"
73-
4. file_write to create or modify data, file_delete to remove data (follow the JSON schema from guide.md)
74-
5. Use the app action tool (REFRESH_*, SYNC_STATE, AGENT_MOVE) to notify the app to reload
75-
76-
### Example - Game move:
77-
1. file_read("apps/{appName}/data/state.json") → get current game state
78-
2. Calculate your move, update the state JSON
79-
3. file_write("apps/{appName}/data/state.json", updatedState) → save
80-
4. {appName}__AGENT_MOVE → app refreshes UI
81-
82-
### Important:
83-
- ALWAYS read meta.yaml and guide.md first before operating on an unfamiliar app.
84-
- After writing data, ALWAYS call the corresponding REFRESH action so the app picks up changes.
85-
- All NAS file operations mentioned in guide.md (read, write, delete, list) map directly to file tools (file_read, file_write, file_delete, file_list). NAS paths like "/articles/xxx.json" translate to "apps/{appName}/data/articles/xxx.json".
86-
87-
## Rules
88-
- Always respond in the same language the user uses.
89-
- When you receive a "[User performed action in ...]" message, it means the user interacted with an app.
90-
- For games, you MUST respond with your own move. Think strategically and play to win, but keep the game fun.
91-
- When the user asks you to generate/draw/create an image, use the generate_image tool with a detailed English prompt.
92-
- When creating content that needs an image, first generate the image with savePath pointing to the app's data directory (e.g. savePath="apps/{appName}/data/images/img-{timestamp}.json"), then reference the relative path "/images/img-{timestamp}.json" in the content's imageUrl field.`;
44+
function buildSystemPrompt(hasImageGen: boolean): string {
45+
return `You are a helpful assistant that can interact with apps on the user's device. Respond in English by default. If the user writes in another language, switch to that language.
46+
47+
For casual conversation (greetings, questions, chat), just reply naturally without using any tools.
48+
49+
When the user wants to interact with an app:
50+
1. Call list_apps to discover available apps and their appName
51+
2. file_read("apps/{appName}/meta.yaml") to learn available actions
52+
3. file_read("apps/{appName}/guide.md") to learn data structure and JSON schema
53+
4. file_list/file_read to explore data in "apps/{appName}/data/"
54+
5. file_write/file_delete to modify data (follow the JSON schema)
55+
6. app_action to notify the app to reload (e.g. REFRESH_*, SYNC_STATE)
56+
57+
NAS paths in guide.md like "/articles/xxx.json" map to "apps/{appName}/data/articles/xxx.json".
58+
After writing data, ALWAYS call app_action with the corresponding REFRESH action.
59+
60+
When you receive "[User performed action in ... (appName: xxx)]", the appName is already provided. Read its meta.yaml to understand available actions, then respond accordingly. For games, respond with your own move — think strategically.${hasImageGen ? '\n\nYou can also use generate_image to create images from text prompts.' : ''}`;
9361
}
9462

9563
const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
@@ -186,7 +154,7 @@ const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
186154
const app = APP_REGISTRY.find((a) => a.appId === action.app_id);
187155
if (!app) return;
188156

189-
const actionMsg = `[User performed action in ${app.displayName}] action_type: ${action.action_type}, params: ${JSON.stringify(action.params || {})}`;
157+
const actionMsg = `[User performed action in ${app.displayName} (appName: ${app.appName})] action_type: ${action.action_type}, params: ${JSON.stringify(action.params || {})}`;
190158
actionQueueRef.current.push(actionMsg);
191159
processActionQueue();
192160
});
@@ -235,26 +203,41 @@ const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
235203
'provider:',
236204
cfg.provider,
237205
);
206+
await seedMetaFiles();
238207
await loadActionsFromMeta();
208+
const hasImageGen = !!imageGenConfigRef.current?.apiKey;
239209
const tools = [
240-
...getToolDefinitions(),
210+
getListAppsToolDefinition(),
211+
getAppActionToolDefinition(),
241212
...getFileToolDefinitions(),
242-
...getImageGenToolDefinitions(),
213+
...(hasImageGen ? getImageGenToolDefinitions() : []),
243214
];
244215
console.info('[ToolLog] ChatPanel: tools passed to chat(), count=', tools.length);
245216
const fullMessages: ChatMessage[] = [
246-
{ role: 'system', content: buildSystemPrompt() },
217+
{ role: 'system', content: buildSystemPrompt(hasImageGen) },
247218
...history,
248219
];
249220

250221
let currentMessages = fullMessages;
251222
let iterations = 0;
252223
const maxIterations = 10;
253224

225+
console.info('[ChatDebug] === START conversation ===');
226+
console.info('[ChatDebug] messages sent to LLM:', JSON.stringify(currentMessages, null, 2));
227+
console.info(
228+
'[ChatDebug] tools:',
229+
tools.map((t) => (t as { function: { name: string } }).function.name),
230+
);
231+
254232
while (iterations < maxIterations) {
255233
iterations++;
234+
console.info(`[ChatDebug] --- iteration ${iterations} ---`);
235+
console.info('[ChatDebug] messages count:', currentMessages.length);
256236
const response = await chat(currentMessages, tools, cfg);
257237

238+
console.info('[ChatDebug] LLM response content:', response.content);
239+
console.info('[ChatDebug] LLM toolCalls:', JSON.stringify(response.toolCalls, null, 2));
240+
258241
if (response.toolCalls.length === 0) {
259242
// No tool calls, just text response
260243
if (response.content) {
@@ -300,6 +283,17 @@ const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
300283
// ignore parse error
301284
}
302285

286+
// list_apps tool
287+
if (tc.function.name === 'list_apps') {
288+
const result = executeListApps();
289+
console.info('[ChatDebug] list_apps result:', result);
290+
currentMessages = [
291+
...currentMessages,
292+
{ role: 'tool', content: result, tool_call_id: tc.id },
293+
];
294+
continue;
295+
}
296+
303297
// File tool calls — direct file operations
304298
if (isFileTool(tc.function.name)) {
305299
addMessage({
@@ -309,6 +303,10 @@ const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
309303
});
310304
try {
311305
const result = await executeFileTool(tc.function.name, params);
306+
console.info(
307+
`[ChatDebug] ${tc.function.name}(${JSON.stringify(params)}) result:`,
308+
result.slice(0, 500),
309+
);
312310
currentMessages = [
313311
...currentMessages,
314312
{ role: 'tool', content: result, tool_call_id: tc.id },
@@ -363,43 +361,60 @@ const ChatPanel: React.FC<{ onClose: () => void; visible?: boolean }> = ({
363361
continue;
364362
}
365363

366-
const parsed = parseToolName(tc.function.name);
367-
if (!parsed) {
368-
currentMessages = [
369-
...currentMessages,
370-
{ role: 'tool', content: 'error: unknown tool', tool_call_id: tc.id },
371-
];
372-
continue;
373-
}
374-
375-
// Regular App tool call
376-
addMessage({
377-
id: String(Date.now()) + tc.id,
378-
role: 'tool',
379-
content: `Calling ${tc.function.name}...`,
380-
});
364+
// app_action tool
365+
if (tc.function.name === 'app_action') {
366+
const resolved = resolveAppAction(params.app_name, params.action_type);
367+
if (typeof resolved === 'string') {
368+
currentMessages = [
369+
...currentMessages,
370+
{ role: 'tool', content: resolved, tool_call_id: tc.id },
371+
];
372+
continue;
373+
}
381374

382-
try {
383-
const result = await dispatchAgentAction({
384-
app_id: parsed.appId,
385-
action_type: parsed.actionType,
386-
params,
375+
addMessage({
376+
id: String(Date.now()) + tc.id,
377+
role: 'tool',
378+
content: `Calling ${params.app_name}/${params.action_type}...`,
387379
});
388380

389-
currentMessages = [
390-
...currentMessages,
391-
{ role: 'tool', content: result, tool_call_id: tc.id },
392-
];
393-
} catch (err) {
394-
currentMessages = [
395-
...currentMessages,
396-
{
397-
role: 'tool',
398-
content: `error: ${err instanceof Error ? err.message : String(err)}`,
399-
tool_call_id: tc.id,
400-
},
401-
];
381+
let actionParams: Record<string, string> = {};
382+
if (params.params) {
383+
try {
384+
actionParams = JSON.parse(params.params);
385+
} catch {
386+
// use empty params
387+
}
388+
}
389+
390+
try {
391+
const result = await dispatchAgentAction({
392+
app_id: resolved.appId,
393+
action_type: resolved.actionType,
394+
params: actionParams,
395+
});
396+
currentMessages = [
397+
...currentMessages,
398+
{ role: 'tool', content: result, tool_call_id: tc.id },
399+
];
400+
} catch (err) {
401+
currentMessages = [
402+
...currentMessages,
403+
{
404+
role: 'tool',
405+
content: `error: ${err instanceof Error ? err.message : String(err)}`,
406+
tool_call_id: tc.id,
407+
},
408+
];
409+
}
410+
continue;
402411
}
412+
413+
// Unknown tool
414+
currentMessages = [
415+
...currentMessages,
416+
{ role: 'tool', content: 'error: unknown tool', tool_call_id: tc.id },
417+
];
403418
}
404419

405420
// Update chat history with tool interactions
@@ -611,8 +626,8 @@ const SettingsModal: React.FC<{
611626
value={igProvider}
612627
onChange={(e) => handleIgProviderChange(e.target.value as ImageGenProvider)}
613628
>
614-
<option value="openai">OpenAI (DALL-E)</option>
615-
<option value="gemini">Gemini (nano-banana-2)</option>
629+
<option value="openai">OpenAI</option>
630+
<option value="gemini">Gemini</option>
616631
</select>
617632
</div>
618633

@@ -643,9 +658,6 @@ const SettingsModal: React.FC<{
643658
value={igModel}
644659
onChange={(e) => setIgModel(e.target.value)}
645660
/>
646-
{igProvider === 'gemini' && igModel === 'gemini-3.1-flash-image-preview' && (
647-
<span className={styles.modelHint}>nano-banana-2</span>
648-
)}
649661
</div>
650662

651663
<div className={styles.field}>

0 commit comments

Comments
 (0)