From 591932602ec5ccda19eba5ae2f6476b6c21956ea Mon Sep 17 00:00:00 2001 From: Joshua Primas Date: Mon, 6 Jul 2026 14:32:30 -0700 Subject: [PATCH 1/4] Allow local image files to be passed to LemonSlice api --- plugins/lemonslice/src/avatar.ts | 76 +++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index e4d6a970b..b48fc16eb 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -14,6 +14,8 @@ import type { Room } from '@livekit/rtc-node'; import { TrackKind } from '@livekit/rtc-node'; import type { VideoGrant } from 'livekit-server-sdk'; import { AccessToken } from 'livekit-server-sdk'; +import { readFileSync } from 'node:fs'; +import { extname } from 'node:path'; import { log } from './log.js'; const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf'; @@ -38,14 +40,19 @@ export class LemonSliceException extends Error { export interface AvatarSessionOptions { /** * The ID of the LemonSlice agent to add to the session. - * Either agentId or agentImageUrl must be provided. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. */ agentId?: string | null; /** * The URL of the image to use as the agent's avatar. - * Either agentId or agentImageUrl must be provided. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. */ agentImageUrl?: string | null; + /** + * A local image file path or Buffer to upload as the agent's avatar. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. + */ + agentImage?: string | Buffer | null; /** * A prompt that subtly influences the avatar's movements and expressions. */ @@ -125,6 +132,8 @@ export interface StartOptions { export class AvatarSession extends voice.AvatarSession { private agentId: string | null; private agentImageUrl: string | null; + private agentImageBytes: Buffer | null; + private agentImageMimeType: string; private agentPrompt: string | null; private idleTimeout: number | null; private extraPayload: Record | null; @@ -146,12 +155,29 @@ export class AvatarSession extends voice.AvatarSession { super(); this.agentId = options.agentId ?? null; this.agentImageUrl = options.agentImageUrl ?? null; + this.agentImageMimeType = 'image/png'; + + if (options.agentImage) { + if (typeof options.agentImage === 'string') { + this.agentImageBytes = readFileSync(options.agentImage); + this.agentImageMimeType = mimeTypeFromExtension(extname(options.agentImage)); + } else { + this.agentImageBytes = options.agentImage; + } + } else { + this.agentImageBytes = null; + } - if (!this.agentId && !this.agentImageUrl) { - throw new LemonSliceException('Missing agentId or agentImageUrl'); + const sourceCount = [this.agentId, this.agentImageUrl, this.agentImageBytes].filter( + Boolean, + ).length; + if (sourceCount === 0) { + throw new LemonSliceException('Missing one of agentId, agentImageUrl, or agentImage'); } - if (this.agentId && this.agentImageUrl) { - throw new LemonSliceException('Only one of agentId or agentImageUrl can be provided'); + if (sourceCount > 1) { + throw new LemonSliceException( + 'Only one of agentId, agentImageUrl, or agentImage can be provided', + ); } this.agentPrompt = options.agentPrompt ?? null; @@ -294,13 +320,29 @@ export class AvatarSession extends voice.AvatarSession { Object.assign(payload, this.extraPayload); } + const headers: Record = { + 'X-API-Key': this.apiKey, + }; + let body: BodyInit; + + if (this.agentImageBytes) { + const formData = new FormData(); + formData.append('payload', JSON.stringify(payload)); + formData.append( + 'image', + new Blob([new Uint8Array(this.agentImageBytes)], { type: this.agentImageMimeType }), + 'image.png', + ); + body = formData; + } else { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify(payload); + } + const response = await fetch(this.apiUrl, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-API-Key': this.apiKey, - }, - body: JSON.stringify(payload), + headers, + body, signal: AbortSignal.timeout(this.connOptions.timeoutMs), }); @@ -336,3 +378,15 @@ export class AvatarSession extends voice.AvatarSession { }); } } + +const MIME_TYPES: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', +}; + +function mimeTypeFromExtension(ext: string): string { + return MIME_TYPES[ext.toLowerCase()] ?? 'image/png'; +} From 6b1d718dd7a69e17f27185389da2b894362551ac Mon Sep 17 00:00:00 2001 From: Joshua Primas Date: Mon, 6 Jul 2026 14:36:16 -0700 Subject: [PATCH 2/4] updated LS README --- plugins/lemonslice/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/lemonslice/README.md b/plugins/lemonslice/README.md index e7f7bd13d..5146da3e6 100644 --- a/plugins/lemonslice/README.md +++ b/plugins/lemonslice/README.md @@ -50,8 +50,9 @@ Set `LEMONSLICE_API_KEY` and `LEMONSLICE_IMAGE_URL` to get up and running. | Option | Type | Description | |--------|------|-------------| -| `agentId` | `string` | The LemonSlice agent ID to use. Either `agentId` or `agentImageUrl` must be provided. | -| `agentImageUrl` | `AvatarImage` | A publicly accessible url to your avatar image. Either `agentId` or `agentImageUrl` must be provided. | +| `agentId` | `string` | The LemonSlice agent ID to use. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | +| `agentImageUrl` | `string` | A publicly accessible url to your avatar image. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | +| `agentImage` | `string \| Buffer` | A local image file path or Buffer to upload as the agent's avatar. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | | `extraPayload` | `Record` | Additional LemonSlice session payload fields to forward to LemonSlice. | | `apiUrl` | `string` | The LemonSlice API URL. Defaults to `LEMONSLICE_API_URL` env var or the default LemonSlice API endpoint. | | `apiKey` | `string` | The LemonSlice API key. Defaults to `LEMONSLICE_API_KEY` env var. | From fa138ec913fe3cc5a9a1999dc8989da02e154ea7 Mon Sep 17 00:00:00 2001 From: Joshua Primas Date: Mon, 6 Jul 2026 15:03:21 -0700 Subject: [PATCH 3/4] mime type logic update --- plugins/lemonslice/README.md | 1 + plugins/lemonslice/src/avatar.ts | 61 +++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/plugins/lemonslice/README.md b/plugins/lemonslice/README.md index 5146da3e6..077f0ae11 100644 --- a/plugins/lemonslice/README.md +++ b/plugins/lemonslice/README.md @@ -53,6 +53,7 @@ Set `LEMONSLICE_API_KEY` and `LEMONSLICE_IMAGE_URL` to get up and running. | `agentId` | `string` | The LemonSlice agent ID to use. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | | `agentImageUrl` | `string` | A publicly accessible url to your avatar image. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | | `agentImage` | `string \| Buffer` | A local image file path or Buffer to upload as the agent's avatar. Exactly one of `agentId`, `agentImageUrl`, or `agentImage` must be provided. | +| `agentImageMimeType` | `string` | MIME type for `agentImage` when provided as a Buffer (e.g. `'image/jpeg'`). Ignored for file paths. Defaults to `'image/png'`. | | `extraPayload` | `Record` | Additional LemonSlice session payload fields to forward to LemonSlice. | | `apiUrl` | `string` | The LemonSlice API URL. Defaults to `LEMONSLICE_API_URL` env var or the default LemonSlice API endpoint. | | `apiKey` | `string` | The LemonSlice API key. Defaults to `LEMONSLICE_API_KEY` env var. | diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index b48fc16eb..5d180741d 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -53,6 +53,11 @@ export interface AvatarSessionOptions { * Exactly one of agentId, agentImageUrl, or agentImage must be provided. */ agentImage?: string | Buffer | null; + /** + * MIME type for the agentImage when provided as a Buffer (e.g. 'image/png'). + * Defaults to 'image/png'. + */ + agentImageMimeType?: string; /** * A prompt that subtly influences the avatar's movements and expressions. */ @@ -155,20 +160,8 @@ export class AvatarSession extends voice.AvatarSession { super(); this.agentId = options.agentId ?? null; this.agentImageUrl = options.agentImageUrl ?? null; - this.agentImageMimeType = 'image/png'; - if (options.agentImage) { - if (typeof options.agentImage === 'string') { - this.agentImageBytes = readFileSync(options.agentImage); - this.agentImageMimeType = mimeTypeFromExtension(extname(options.agentImage)); - } else { - this.agentImageBytes = options.agentImage; - } - } else { - this.agentImageBytes = null; - } - - const sourceCount = [this.agentId, this.agentImageUrl, this.agentImageBytes].filter( + const sourceCount = [this.agentId, this.agentImageUrl, options.agentImage].filter( Boolean, ).length; if (sourceCount === 0) { @@ -180,6 +173,20 @@ export class AvatarSession extends voice.AvatarSession { ); } + this.agentImageMimeType = 'image/png'; + if (options.agentImage) { + if (typeof options.agentImage === 'string') { + this.agentImageMimeType = mimeTypeFromExtension(extname(options.agentImage)); + this.agentImageBytes = readFileSync(options.agentImage); + } else { + this.agentImageMimeType = options.agentImageMimeType ?? 'image/png'; + validateMimeType(this.agentImageMimeType); + this.agentImageBytes = options.agentImage; + } + } else { + this.agentImageBytes = null; + } + this.agentPrompt = options.agentPrompt ?? null; this.idleTimeout = options.idleTimeout ?? null; this.extraPayload = options.extraPayload ?? null; @@ -331,7 +338,7 @@ export class AvatarSession extends voice.AvatarSession { formData.append( 'image', new Blob([new Uint8Array(this.agentImageBytes)], { type: this.agentImageMimeType }), - 'image.png', + `image${extensionFromMimeType(this.agentImageMimeType)}`, ); body = formData; } else { @@ -383,10 +390,32 @@ const MIME_TYPES: Record = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', - '.gif': 'image/gif', '.webp': 'image/webp', }; +const SUPPORTED_MIME_TYPES = new Set(Object.values(MIME_TYPES)); + function mimeTypeFromExtension(ext: string): string { - return MIME_TYPES[ext.toLowerCase()] ?? 'image/png'; + const mime = MIME_TYPES[ext.toLowerCase()]; + if (!mime) { + throw new LemonSliceException( + `Unsupported image extension '${ext}'. Supported: ${Object.keys(MIME_TYPES).join(', ')}`, + ); + } + return mime; +} + +function validateMimeType(mime: string): void { + if (!SUPPORTED_MIME_TYPES.has(mime)) { + throw new LemonSliceException( + `Unsupported MIME type '${mime}'. Supported: ${[...SUPPORTED_MIME_TYPES].join(', ')}`, + ); + } +} + +function extensionFromMimeType(mime: string): string { + for (const [ext, type] of Object.entries(MIME_TYPES)) { + if (type === mime) return ext; + } + return '.png'; } From 6f0c6622ddcf470f9d66a061a0d0985028bffe39 Mon Sep 17 00:00:00 2001 From: Joshua Primas Date: Tue, 7 Jul 2026 11:34:30 -0700 Subject: [PATCH 4/4] added changeset --- .changeset/chubby-months-stop.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/chubby-months-stop.md diff --git a/.changeset/chubby-months-stop.md b/.changeset/chubby-months-stop.md new file mode 100644 index 000000000..ddefdf3e0 --- /dev/null +++ b/.changeset/chubby-months-stop.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-lemonslice': patch +--- + +Add the ability for users to provide a local image to the LemonSlice transport to be used as the avatar image.