Skip to content

Commit 1c25b06

Browse files
tjbcktjbck
committed
refac
1 parent 36c3fc5 commit 1c25b06

4 files changed

Lines changed: 111 additions & 26 deletions

File tree

‎backend/open_webui/utils/files.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def get_audio_url_from_base64(request, base64_audio_string, metadata, user):
148148

149149

150150
def get_file_url_from_base64(request, base64_file_string, metadata, user):
151-
if 'data:image/png;base64' in base64_file_string:
151+
if BASE64_IMAGE_URL_PREFIX.match(base64_file_string):
152152
return get_image_url_from_base64(request, base64_file_string, metadata, user)
153153
elif 'data:audio/wav;base64' in base64_file_string:
154154
return get_audio_url_from_base64(request, base64_file_string, metadata, user)

‎backend/open_webui/utils/middleware.py‎

Lines changed: 61 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1049,6 +1049,13 @@ def process_tool_result(
10491049

10501050
tool_result_files = []
10511051

1052+
# Detect base64 image data URIs from tool results (e.g. binary image
1053+
# responses from execute_tool_server). Move the data URI to
1054+
# tool_result_files and replace tool_result with a text summary.
1055+
if isinstance(tool_result, str) and tool_result.startswith('data:image/'):
1056+
tool_result_files.append({'type': 'image', 'url': tool_result})
1057+
tool_result = f'{tool_function_name}: Image file read successfully.'
1058+
10521059
if isinstance(tool_result, list):
10531060
if tool_type == 'mcp': # MCP
10541061
tool_response = []
@@ -4181,19 +4188,27 @@ async def flush_pending_delta_data(threshold: int = 0):
41814188
break
41824189

41834190
for result in results:
4191+
output_parts = [{'type': 'input_text', 'text': result.get('content', '')}]
4192+
4193+
# Separate image data URIs (for LLM via input_image) from
4194+
# other files (for frontend display via files attribute).
4195+
display_files = []
4196+
for file_item in result.get('files', []):
4197+
if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'):
4198+
# LLM-only: add as input_image part (invisible to serialize_output)
4199+
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
4200+
else:
4201+
# Frontend display (MCP images, audio, etc.)
4202+
display_files.append(file_item)
4203+
41844204
output.append(
41854205
{
41864206
'type': 'function_call_output',
41874207
'id': output_id('fco'),
41884208
'call_id': result.get('tool_call_id', ''),
4189-
'output': [
4190-
{
4191-
'type': 'input_text',
4192-
'text': result.get('content', ''),
4193-
}
4194-
],
4209+
'output': output_parts,
41954210
'status': 'completed',
4196-
**({'files': result.get('files')} if result.get('files') else {}),
4211+
**({'files': display_files} if display_files else {}),
41974212
**({'embeds': result.get('embeds')} if result.get('embeds') else {}),
41984213
}
41994214
)
@@ -4262,12 +4277,23 @@ async def flush_pending_delta_data(threshold: int = 0):
42624277
)
42634278
tool_call_sources.clear()
42644279

4280+
# Strip input_image parts (large base64 data URIs) from the
4281+
# output sent to the frontend — they're only for LLM consumption
4282+
# via convert_output_to_messages.
4283+
frontend_output = []
4284+
for item in output:
4285+
if item.get('type') == 'function_call_output':
4286+
parts = item.get('output', [])
4287+
if any(p.get('type') == 'input_image' for p in parts):
4288+
item = {**item, 'output': [p for p in parts if p.get('type') != 'input_image']}
4289+
frontend_output.append(item)
4290+
42654291
await event_emitter(
42664292
{
42674293
'type': 'chat:completion',
42684294
'data': {
42694295
'content': serialize_output(output),
4270-
'output': output,
4296+
'output': frontend_output,
42714297
},
42724298
}
42734299
)
@@ -4287,11 +4313,35 @@ async def flush_pending_delta_data(threshold: int = 0):
42874313
)
42884314
new_form_data['previous_response_id'] = last_response_id
42894315
else:
4316+
tool_messages = convert_output_to_messages(output, raw=True)
4317+
4318+
# Chat Completions providers don't support multimodal
4319+
# tool messages. Extract images into a user message.
4320+
image_urls = []
4321+
for message in tool_messages:
4322+
if message.get('role') == 'tool' and isinstance(message.get('content'), list):
4323+
text_parts = []
4324+
for part in message['content']:
4325+
if part.get('type') == 'input_text':
4326+
text_parts.append(part.get('text', ''))
4327+
elif part.get('type') == 'input_image':
4328+
image_urls.append(part.get('image_url', ''))
4329+
message['content'] = ''.join(text_parts)
4330+
42904331
new_form_data['messages'] = [
42914332
*form_data['messages'],
4292-
*convert_output_to_messages(output, raw=True),
4333+
*tool_messages,
42934334
]
42944335

4336+
if image_urls:
4337+
new_form_data['messages'].append({
4338+
'role': 'user',
4339+
'content': [
4340+
{'type': 'text', 'text': 'Here are the images from the tool results above. Please analyze them.'},
4341+
*[{'type': 'image_url', 'image_url': {'url': url}} for url in image_urls],
4342+
],
4343+
})
4344+
42954345
res = await generate_chat_completion(
42964346
request,
42974347
new_form_data,
@@ -4416,7 +4466,7 @@ def restricted_import(name, globals=None, locals=None, fromlist=(), level=0):
44164466
if isinstance(stdout, str):
44174467
stdoutLines = stdout.split('\n')
44184468
for idx, line in enumerate(stdoutLines):
4419-
if 'data:image/png;base64' in line:
4469+
if re.match(r'data:image/\w+;base64', line):
44204470
image_url = get_image_url_from_base64(
44214471
request,
44224472
line,
@@ -4433,7 +4483,7 @@ def restricted_import(name, globals=None, locals=None, fromlist=(), level=0):
44334483
if isinstance(result, str):
44344484
resultLines = result.split('\n')
44354485
for idx, line in enumerate(resultLines):
4436-
if 'data:image/png;base64' in line:
4486+
if re.match(r'data:image/\w+;base64', line):
44374487
image_url = get_image_url_from_base64(
44384488
request,
44394489
line,

‎backend/open_webui/utils/misc.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -196,21 +196,39 @@ def flush_pending():
196196
# Flush any pending content/tool_calls before adding tool result
197197
flush_pending()
198198

199-
# Extract text from output content parts
199+
# Extract text and images from output content parts
200200
output_parts = item.get('output', [])
201201
content = ''
202+
image_urls = []
202203
for part in output_parts:
203204
if part.get('type') == 'input_text':
204205
output_text = part.get('text', '')
205206
content += str(output_text) if not isinstance(output_text, str) else output_text
206-
207-
messages.append(
208-
{
209-
'role': 'tool',
210-
'tool_call_id': item.get('call_id', ''),
211-
'content': content,
212-
}
213-
)
207+
elif part.get('type') == 'input_image':
208+
url = part.get('image_url', '')
209+
if url:
210+
image_urls.append(url)
211+
212+
if image_urls:
213+
# Multimodal tool content with image(s)
214+
messages.append(
215+
{
216+
'role': 'tool',
217+
'tool_call_id': item.get('call_id', ''),
218+
'content': [
219+
{'type': 'input_text', 'text': content},
220+
*[{'type': 'input_image', 'image_url': url} for url in image_urls],
221+
],
222+
}
223+
)
224+
else:
225+
messages.append(
226+
{
227+
'role': 'tool',
228+
'tool_call_id': item.get('call_id', ''),
229+
'content': content,
230+
}
231+
)
214232

215233
elif item_type == 'reasoning':
216234
if raw:

‎backend/open_webui/utils/tools.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import base64
12
import inspect
23
import logging
34
import re
@@ -1236,9 +1237,13 @@ async def execute_tool_server(
12361237
if param_name in params:
12371238
if param_in == 'path':
12381239
path_params[param_name] = params[param_name]
1239-
elif param_in == 'query':
1240-
if params[param_name] is not None:
1241-
query_params[param_name] = params[param_name]
1240+
if param_in == 'query':
1241+
value = params[param_name]
1242+
# Skip empty values for optional params (LLMs sometimes
1243+
# pass "" instead of omitting optional parameters).
1244+
if value is None or (value == '' and not param.get('required')):
1245+
continue
1246+
query_params[param_name] = value
12421247

12431248
final_url = f'{url.rstrip("/")}{route_path}'
12441249
for key, value in path_params.items():
@@ -1273,7 +1278,13 @@ async def execute_tool_server(
12731278
try:
12741279
response_data = await response.json()
12751280
except Exception:
1276-
response_data = await response.text()
1281+
content_type = response.headers.get('Content-Type', '').split(';')[0].strip()
1282+
if content_type.startswith('text/') or not content_type:
1283+
response_data = await response.text()
1284+
else:
1285+
raw = await response.read()
1286+
b64 = base64.b64encode(raw).decode()
1287+
response_data = f'data:{content_type};base64,{b64}'
12771288

12781289
response_headers = response.headers
12791290
return (response_data, response_headers)
@@ -1292,7 +1303,13 @@ async def execute_tool_server(
12921303
try:
12931304
response_data = await response.json()
12941305
except Exception:
1295-
response_data = await response.text()
1306+
content_type = response.headers.get('Content-Type', '').split(';')[0].strip()
1307+
if content_type.startswith('text/') or not content_type:
1308+
response_data = await response.text()
1309+
else:
1310+
raw = await response.read()
1311+
b64 = base64.b64encode(raw).decode()
1312+
response_data = f'data:{content_type};base64,{b64}'
12961313

12971314
response_headers = response.headers
12981315
return (response_data, response_headers)

0 commit comments

Comments
 (0)