Skip to content

Commit a508405

Browse files
chriscarrollsmithchriscarrollsmith
committed
Support image upload with "vision" purpose in chat
1 parent 479f322 commit a508405

7 files changed

Lines changed: 496 additions & 35 deletions

File tree

‎routers/chat.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from typing import AsyncGenerator, Dict, Any, Callable, cast
88
from pydantic import ValidationError
99
from fastapi.templating import Jinja2Templates
10-
from fastapi import APIRouter, Form, Depends, Request
10+
from fastapi import APIRouter, Form, Depends, Request, UploadFile, File
1111
from fastapi.responses import StreamingResponse, HTMLResponse
1212
from openai.types.responses import (
1313
ResponseCreatedEvent, ResponseOutputItemAddedEvent,
@@ -57,23 +57,42 @@ async def send_message(
5757
request: Request,
5858
conversation_id: str,
5959
userInput: str = Form(...),
60+
image: UploadFile | None = File(None),
6061
client: AsyncOpenAI = Depends(lambda: AsyncOpenAI())
6162
) -> HTMLResponse:
63+
# Build multimodal content array
64+
content: list[dict[str, str]] = [{
65+
"type": "input_text",
66+
"text": f"System: Today's date is {datetime.today().strftime('%Y-%m-%d')}\n{userInput}"
67+
}]
68+
69+
# If an image was uploaded, send it to OpenAI and add to content
70+
image_file_id: str | None = None
71+
if image and image.filename and image.size:
72+
image_bytes = await image.read()
73+
if image_bytes:
74+
openai_file = await client.files.create(
75+
file=(image.filename, image_bytes),
76+
purpose="vision"
77+
)
78+
image_file_id = openai_file.id
79+
content.append({
80+
"type": "input_image",
81+
"file_id": image_file_id,
82+
})
83+
6284
# Create a new conversation item for the user's message
6385
await client.conversations.items.create(
6486
conversation_id=conversation_id,
6587
items=[{
6688
"type": "message",
6789
"role": "user",
68-
"content": [{
69-
"type": "input_text",
70-
"text": f"System: Today's date is {datetime.today().strftime('%Y-%m-%d')}\n{userInput}"
71-
}]
90+
"content": content
7291
}]
7392
)
7493

7594
user_message_html = templates.get_template("components/user-message.html").render(
76-
request=request, user_input=userInput
95+
request=request, user_input=userInput, image_file_id=image_file_id
7796
)
7897
assistant_run_html = templates.get_template("components/assistant-run.html").render(
7998
request=request, conversation_id=conversation_id

‎static/stream-md.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,3 +259,26 @@ window.removeNetworkError = function() {
259259
console.warn('removeNetworkError failed:', e);
260260
}
261261
};
262+
263+
// Image upload preview helpers
264+
window.previewImage = function(input) {
265+
const preview = document.getElementById('imagePreview');
266+
const previewImg = document.getElementById('imagePreviewImg');
267+
if (input.files && input.files[0]) {
268+
const reader = new FileReader();
269+
reader.onload = function(e) {
270+
previewImg.src = e.target.result;
271+
preview.style.display = 'flex';
272+
};
273+
reader.readAsDataURL(input.files[0]);
274+
}
275+
};
276+
277+
window.clearImagePreview = function() {
278+
const preview = document.getElementById('imagePreview');
279+
const previewImg = document.getElementById('imagePreviewImg');
280+
const imageInput = document.getElementById('imageInput');
281+
if (preview) preview.style.display = 'none';
282+
if (previewImg) previewImg.src = '';
283+
if (imageInput) imageInput.value = '';
284+
};

‎static/styles.css‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,13 +407,83 @@ pre {
407407

408408
.inputForm {
409409
display: flex;
410-
align-items: flex-end;
410+
flex-direction: column;
411411
width: 100%;
412412
padding: 10px;
413413
padding-bottom: 40px;
414414
order: 1;
415415
}
416416

417+
.inputRow {
418+
display: flex;
419+
align-items: flex-end;
420+
width: 100%;
421+
}
422+
423+
/* Image upload button */
424+
.imageUploadLabel {
425+
display: flex;
426+
align-items: center;
427+
justify-content: center;
428+
width: 40px;
429+
height: calc(1em + 32px + 4px);
430+
cursor: pointer;
431+
color: #666;
432+
flex-shrink: 0;
433+
border-radius: 50%;
434+
transition: color 0.2s;
435+
}
436+
437+
.imageUploadLabel:hover {
438+
color: #000;
439+
}
440+
441+
.imageUploadInput {
442+
display: none;
443+
}
444+
445+
/* Image preview above the input row */
446+
.imagePreview {
447+
display: flex;
448+
align-items: center;
449+
gap: 8px;
450+
padding: 8px;
451+
margin-bottom: 8px;
452+
background: #f5f5f5;
453+
border-radius: 8px;
454+
width: fit-content;
455+
}
456+
457+
.imagePreview img {
458+
max-height: 80px;
459+
max-width: 120px;
460+
border-radius: 6px;
461+
object-fit: cover;
462+
}
463+
464+
.imagePreviewRemove {
465+
background: none;
466+
border: none;
467+
font-size: 1.2em;
468+
cursor: pointer;
469+
color: #999;
470+
padding: 4px 8px;
471+
line-height: 1;
472+
}
473+
474+
.imagePreviewRemove:hover {
475+
color: #333;
476+
}
477+
478+
/* Thumbnail in user messages */
479+
.userImageThumb {
480+
max-height: 120px;
481+
max-width: 200px;
482+
border-radius: 8px;
483+
margin-bottom: 8px;
484+
display: block;
485+
}
486+
417487
.input {
418488
flex-grow: 1;
419489
padding: 16px 24px;

‎templates/components/user-message.html‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,7 @@
11
<!-- user-message.html -->
2-
<div class="userMessage">{{ user_input }}</div>
2+
<div class="userMessage">
3+
{% if image_file_id %}
4+
<img src="/files/{{ image_file_id }}/content" alt="Uploaded image" class="userImageThumb" />
5+
{% endif %}
6+
{{ user_input }}
7+
</div>

‎templates/index.html‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -24,33 +24,51 @@
2424
{% endfor %}
2525
</div>
2626
<form id="chatForm" class="inputForm clearfix"
27-
hx-on::after-request="this.reset()"
28-
hx-on::before-request="removeNetworkError(); disableSendButton()">
29-
<textarea
30-
class="input"
31-
name="userInput"
32-
placeholder="Enter your question"
33-
id="userInput"
34-
autocomplete="off"
35-
rows="1" {# Start with one row, will expand with CSS #}
36-
oninput="this.style.height = 'auto'; this.style.height = (this.scrollHeight) + 'px';"
37-
hx-on:keydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.querySelector('button[type=submit]').click(); }"
38-
required
39-
></textarea>
40-
<button
41-
id="sendButton"
42-
type="submit"
43-
class="button"
44-
hx-post="/chat/{{ conversation_id }}/send"
45-
hx-target="#messages"
46-
hx-swap="beforeend"
47-
{% if inputDisabled %}disabled{% endif %}
48-
>
49-
<span class="button__text">Send</span>
50-
<span class="button__loader" style="display: none;">
51-
<span class="loading-dot"></span><span class="loading-dot"></span><span class="loading-dot"></span>
52-
</span>
53-
</button>
27+
hx-on::after-request="this.reset(); clearImagePreview();"
28+
hx-on::before-request="removeNetworkError(); disableSendButton()"
29+
hx-encoding="multipart/form-data">
30+
<div id="imagePreview" class="imagePreview" style="display:none;">
31+
<img id="imagePreviewImg" src="" alt="Preview" />
32+
<button type="button" class="imagePreviewRemove" onclick="clearImagePreview()">&times;</button>
33+
</div>
34+
<div class="inputRow">
35+
<label class="imageUploadLabel" title="Attach image">
36+
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/><circle cx="8.5" cy="8.5" r="1.5"/><polyline points="21 15 16 10 5 21"/></svg>
37+
<input
38+
type="file"
39+
name="image"
40+
id="imageInput"
41+
accept="image/png,image/jpeg,image/webp,image/gif"
42+
class="imageUploadInput"
43+
onchange="previewImage(this)"
44+
/>
45+
</label>
46+
<textarea
47+
class="input"
48+
name="userInput"
49+
placeholder="Enter your question"
50+
id="userInput"
51+
autocomplete="off"
52+
rows="1" {# Start with one row, will expand with CSS #}
53+
oninput="this.style.height = 'auto'; this.style.height = (this.scrollHeight) + 'px';"
54+
hx-on:keydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.querySelector('button[type=submit]').click(); }"
55+
required
56+
></textarea>
57+
<button
58+
id="sendButton"
59+
type="submit"
60+
class="button"
61+
hx-post="/chat/{{ conversation_id }}/send"
62+
hx-target="#messages"
63+
hx-swap="beforeend"
64+
{% if inputDisabled %}disabled{% endif %}
65+
>
66+
<span class="button__text">Send</span>
67+
<span class="button__loader" style="display: none;">
68+
<span class="loading-dot"></span><span class="loading-dot"></span><span class="loading-dot"></span>
69+
</span>
70+
</button>
71+
</div>
5472
</form>
5573
</div>
5674
{% endblock %}

0 commit comments

Comments
 (0)