Skip to content

Commit 4f13cb7

Browse files
foldlfoldl
authored
webui: support video files as input (ggml-org#22830)
1 parent b64739e commit 4f13cb7

36 files changed

Lines changed: 310 additions & 28 deletions

‎tools/ui/src/app.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import type {
3939
DatabaseMessage,
4040
DatabaseMessageExtra,
4141
DatabaseMessageExtraAudioFile,
42+
DatabaseMessageExtraVideoFile,
4243
DatabaseMessageExtraImageFile,
4344
DatabaseMessageExtraTextFile,
4445
DatabaseMessageExtraPdfFile,
@@ -102,6 +103,7 @@ declare global {
102103
DatabaseMessage,
103104
DatabaseMessageExtra,
104105
DatabaseMessageExtraAudioFile,
106+
DatabaseMessageExtraVideoFile,
105107
DatabaseMessageExtraImageFile,
106108
DatabaseMessageExtraTextFile,
107109
DatabaseMessageExtraPdfFile,

‎tools/ui/src/lib/components/app/badges/BadgesModality.svelte‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<script lang="ts">
2-
import { Eye, Mic } from '@lucide/svelte';
2+
import { Eye, Mic, Video } from '@lucide/svelte';
33
import { ModelModality } from '$lib/enums';
44
55
interface Props {
@@ -11,7 +11,7 @@
1111
</script>
1212

1313
{#each modalities as modality (modality)}
14-
{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO}
14+
{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO || modality === ModelModality.VIDEO}
1515
<span
1616
class={[
1717
'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
@@ -21,7 +21,11 @@
2121
{#if modality === ModelModality.VISION}
2222
<Eye class="h-3 w-3" />
2323

24-
Vision
24+
Vision (Image)
25+
{:else if modality === ModelModality.VIDEO}
26+
<Video class="h-3 w-3" />
27+
28+
Vision (Video)
2529
{:else}
2630
<Mic class="h-3 w-3" />
2731

‎tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
<script lang="ts">
2-
import { X } from '@lucide/svelte';
2+
import { X, Music, Video } from '@lucide/svelte';
33
import {
44
formatFileSize,
55
getFileTypeLabel,
66
getPreviewText,
77
isPdfFile,
8+
isAudioFile,
9+
isVideoFile,
810
isTextFile
911
} from '$lib/utils';
1012
import { ActionIcon } from '$lib/components/app';
@@ -38,6 +40,8 @@
3840
}: Props = $props();
3941
4042
let isPdf = $derived(isPdfFile(attachment, uploadedFile));
43+
let isAudio = $derived(isAudioFile(attachment, uploadedFile));
44+
let isVideo = $derived(isVideoFile(attachment, uploadedFile));
4145
let isPdfWithContent = $derived(isPdf && !!textContent);
4246
4347
let isText = $derived(isTextFile(attachment, uploadedFile));
@@ -102,7 +106,13 @@
102106
<div
103107
class="flex h-8 w-8 items-center justify-center rounded bg-primary/10 text-xs font-medium text-primary"
104108
>
105-
{fileTypeLabel}
109+
{#if isAudio}
110+
<Music class="h-4 w-4 text-white/70" />
111+
{:else if isVideo}
112+
<Video class="h-4 w-4 text-white/70" />
113+
{:else}
114+
{fileTypeLabel}
115+
{/if}
106116
</div>
107117
{/snippet}
108118

‎tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview.svelte‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
getAttachmentDisplayItems,
1313
getLanguageFromFilename,
1414
isAudioFile,
15+
isVideoFile,
1516
isImageFile,
1617
isMcpPrompt,
1718
isMcpResource,
@@ -29,6 +30,7 @@
2930
textContent?: string;
3031
isImage: boolean;
3132
isAudio: boolean;
33+
isVideo: boolean;
3234
}
3335
3436
interface Props {
@@ -54,7 +56,8 @@
5456
(item): PreviewItem => ({
5557
...item,
5658
isImage: isImageFile(item.attachment, item.uploadedFile),
57-
isAudio: isAudioFile(item.attachment, item.uploadedFile)
59+
isAudio: isAudioFile(item.attachment, item.uploadedFile),
60+
isVideo: isVideoFile(item.attachment, item.uploadedFile)
5861
})
5962
)
6063
);
@@ -102,6 +105,9 @@
102105
let isAudio = $derived(
103106
currentItem ? isAudioFile(currentItem.attachment, currentItem.uploadedFile) : false
104107
);
108+
let isVideo = $derived(
109+
currentItem ? isVideoFile(currentItem.attachment, currentItem.uploadedFile) : false
110+
);
105111
let isImage = $derived(
106112
currentItem ? isImageFile(currentItem.attachment, currentItem.uploadedFile) : false
107113
);
@@ -148,6 +154,20 @@
148154
: null
149155
);
150156
157+
let videoSrc = $derived(
158+
isVideo && currentItem
159+
? (currentItem.uploadedFile?.preview ??
160+
(currentItem.attachment &&
161+
'mimeType' in currentItem.attachment &&
162+
'base64Data' in currentItem.attachment
163+
? createBase64DataUrl(
164+
currentItem.attachment.mimeType,
165+
currentItem.attachment.base64Data
166+
)
167+
: null))
168+
: null
169+
);
170+
151171
export function prev() {
152172
currentIndex = currentIndex > 0 ? currentIndex - 1 : allItems.length - 1;
153173
}
@@ -173,11 +193,13 @@
173193
{currentItem}
174194
{isImage}
175195
{isAudio}
196+
{isVideo}
176197
{isPdf}
177198
{isText}
178199
{displayPreview}
179200
{displayTextContent}
180201
{audioSrc}
202+
{videoSrc}
181203
{language}
182204
{hasVisionModality}
183205
{activeModelId}

‎tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItem.svelte‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
11
<script lang="ts">
22
import type { ChatAttachmentDisplayItem } from '$lib/types';
3-
import { Image, Music, FileText, FileIcon } from '@lucide/svelte';
3+
import { Image, Music, Video, FileText, FileIcon } from '@lucide/svelte';
44
import ChatAttachmentsPreviewCurrentItemPdf from './ChatAttachmentsPreviewCurrentItemPdf.svelte';
55
import ChatAttachmentsPreviewCurrentItemImage from './ChatAttachmentsPreviewCurrentItemImage.svelte';
66
import ChatAttachmentsPreviewCurrentItemAudio from './ChatAttachmentsPreviewCurrentItemAudio.svelte';
7+
import ChatAttachmentsPreviewCurrentItemVideo from './ChatAttachmentsPreviewCurrentItemVideo.svelte';
78
import ChatAttachmentsPreviewCurrentItemText from './ChatAttachmentsPreviewCurrentItemText.svelte';
89
import ChatAttachmentsPreviewCurrentItemUnavailable from './ChatAttachmentsPreviewCurrentItemUnavailable.svelte';
910
1011
interface Props {
1112
currentItem: ChatAttachmentDisplayItem | null;
1213
isImage: boolean;
1314
isAudio: boolean;
15+
isVideo: boolean;
1416
isPdf: boolean;
1517
isText: boolean;
1618
displayPreview: string | undefined;
1719
displayTextContent: string | undefined;
1820
audioSrc: string | null;
21+
videoSrc: string | null;
1922
language: string;
2023
hasVisionModality: boolean;
2124
activeModelId?: string;
@@ -25,21 +28,25 @@
2528
currentItem,
2629
isImage,
2730
isAudio,
31+
isVideo,
2832
isPdf,
2933
isText,
3034
displayPreview,
3135
displayTextContent,
3236
audioSrc,
37+
videoSrc,
3338
language,
3439
hasVisionModality,
3540
activeModelId
3641
}: Props = $props();
3742
3843
let IconComponent = $derived(
39-
isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : FileIcon
44+
isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : isVideo ? Video : FileIcon
4045
);
4146
42-
let isUnavailable = $derived(!isPdf && !isImage && !(isText && displayTextContent) && !isAudio);
47+
let isUnavailable = $derived(
48+
!isPdf && !isImage && !(isText && displayTextContent) && !isAudio && !isVideo
49+
);
4350
</script>
4451

4552
{#if currentItem}
@@ -58,6 +65,8 @@
5865
<ChatAttachmentsPreviewCurrentItemText {displayTextContent} {language} />
5966
{:else if isAudio}
6067
<ChatAttachmentsPreviewCurrentItemAudio {currentItem} {audioSrc} />
68+
{:else if isVideo}
69+
<ChatAttachmentsPreviewCurrentItemVideo {currentItem} {videoSrc} />
6170
{:else if isUnavailable}
6271
<ChatAttachmentsPreviewCurrentItemUnavailable {IconComponent} />
6372
{/if}

‎tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemVideo.svelte‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<script lang="ts">
2+
import { Video } from '@lucide/svelte';
3+
4+
interface Props {
5+
currentItem: { name?: string } | null;
6+
videoSrc: string | null;
7+
}
8+
9+
let { currentItem, videoSrc }: Props = $props();
10+
</script>
11+
12+
<div class="flex flex-1 items-center justify-center p-8">
13+
<div class="w-full max-w-md text-center">
14+
<Video class="mx-auto mb-4 h-16 w-16 text-white/50" />
15+
16+
{#if videoSrc}
17+
<video controls class="mb-4 w-full" src={videoSrc}>
18+
Your browser does not support the video element.
19+
</video>
20+
{:else}
21+
<p class="mb-4 text-white/70">Video preview not available</p>
22+
{/if}
23+
24+
<p class="text-sm text-white/50">{currentItem?.name || 'Video'}</p>
25+
</div>
26+
</div>

‎tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
<script lang="ts">
2-
import { Music, FileText } from '@lucide/svelte';
2+
import { Music, Video, FileText } from '@lucide/svelte';
33
import { HorizontalScrollCarousel } from '$lib/components/app/misc';
44
55
interface PreviewItem {
66
id: string;
77
name: string;
88
isImage: boolean;
99
isAudio: boolean;
10+
isVideo: boolean;
1011
preview?: string;
1112
}
1213
@@ -49,6 +50,8 @@
4950
>
5051
{#if item.isAudio}
5152
<Music class="h-4 w-4 text-white/70" />
53+
{:else if item.isVideo}
54+
<Video class="h-4 w-4 text-white/70" />
5255
{:else}
5356
<FileText class="h-4 w-4 text-white/70" />
5457
{/if}

‎tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
class?: string;
2424
disabled?: boolean;
2525
hasAudioModality?: boolean;
26+
hasVideoModality?: boolean;
2627
hasVisionModality?: boolean;
2728
hasMcpPromptsSupport?: boolean;
2829
hasMcpResourcesSupport?: boolean;
@@ -37,6 +38,7 @@
3738
class: className = '',
3839
disabled = false,
3940
hasAudioModality = false,
41+
hasVideoModality = false,
4042
hasVisionModality = false,
4143
hasMcpPromptsSupport = false,
4244
hasMcpResourcesSupport = false,
@@ -58,6 +60,7 @@
5860
() => ({
5961
hasVisionModality,
6062
hasAudioModality,
63+
hasVideoModality,
6164
hasMcpPromptsSupport,
6265
hasMcpResourcesSupport
6366
}),

‎tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
class?: string;
2020
disabled?: boolean;
2121
hasAudioModality?: boolean;
22+
hasVideoModality?: boolean;
2223
hasVisionModality?: boolean;
2324
hasMcpPromptsSupport?: boolean;
2425
hasMcpResourcesSupport?: boolean;
@@ -34,6 +35,7 @@
3435
disabled = false,
3536
hasAudioModality = false,
3637
hasVisionModality = false,
38+
hasVideoModality = false,
3739
hasMcpPromptsSupport = false,
3840
hasMcpResourcesSupport = false,
3941
onFileUpload,
@@ -49,6 +51,7 @@
4951
() => ({
5052
hasVisionModality,
5153
hasAudioModality,
54+
hasVideoModality,
5255
hasMcpPromptsSupport,
5356
hasMcpResourcesSupport
5457
}),

‎tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
interface Props {
88
disabled?: boolean;
99
hasAudioModality?: boolean;
10+
hasVideoModality?: boolean;
1011
hasMcpPromptsSupport?: boolean;
1112
hasMcpResourcesSupport?: boolean;
1213
hasVisionModality?: boolean;
@@ -20,6 +21,7 @@
2021
let {
2122
disabled = false,
2223
hasAudioModality = false,
24+
hasVideoModality = false,
2325
hasMcpPromptsSupport = false,
2426
hasMcpResourcesSupport = false,
2527
hasVisionModality = false,
@@ -37,6 +39,7 @@
3739
<ChatFormActionAddSheet
3840
{disabled}
3941
{hasAudioModality}
42+
{hasVideoModality}
4043
{hasVisionModality}
4144
{hasMcpPromptsSupport}
4245
{hasMcpResourcesSupport}
@@ -52,6 +55,7 @@
5255
<ChatFormActionAddDropdown
5356
{disabled}
5457
{hasAudioModality}
58+
{hasVideoModality}
5559
{hasVisionModality}
5660
{hasMcpPromptsSupport}
5761
{hasMcpResourcesSupport}

0 commit comments

Comments
 (0)