diff --git a/apps/web/src/lib/bot/agent-runner.ts b/apps/web/src/lib/bot/agent-runner.ts index 4b614bdcb..9367d60a5 100644 --- a/apps/web/src/lib/bot/agent-runner.ts +++ b/apps/web/src/lib/bot/agent-runner.ts @@ -23,6 +23,7 @@ import { buildSessionUrl } from '@/lib/cloud-agent-next/session-url'; import { APP_URL } from '@/lib/constants'; import { FEATURE_HEADER } from '@/lib/feature-detection'; import { ownerFromIntegration } from '@/lib/integrations/core/owner'; +import type { Images } from '@/lib/images-schema'; import { formatGitHubRepositoriesForPrompt, getGitHubRepositoryContext, @@ -60,6 +61,8 @@ type RunBotAgentParams = { user: User; botRequestId: string | undefined; prompt: string; + /** Pre-uploaded image attachments from the user's message (already in R2). */ + images?: Images; completedStepCount?: number; initialSteps?: BotRequestStep[]; onSessionReady?: (params: { @@ -262,6 +265,8 @@ export async function runBotAgent(params: RunBotAgentParams): Promise { @@ -298,6 +303,7 @@ This tool returns an acknowledgement immediately. The final Cloud Agent result w prSignature, chatPlatform, currentStep, + images: params.images, } ); @@ -332,7 +338,13 @@ This tool returns an acknowledgement immediately. The final Cloud Agent result w }, }); - const result = await agent.generate({ prompt: params.prompt }); + const imageCount = params.images?.files.length ?? 0; + const promptWithImageContext = + imageCount > 0 + ? `${params.prompt}\n\n[The user attached ${imageCount} image${imageCount > 1 ? 's' : ''} to this message. The image${imageCount > 1 ? 's are' : ' is'} automatically forwarded to any Cloud Agent session you spawn.]` + : params.prompt; + + const result = await agent.generate({ prompt: promptWithImageContext }); return { finalText: result.text, diff --git a/apps/web/src/lib/bot/images.ts b/apps/web/src/lib/bot/images.ts new file mode 100644 index 000000000..5d88b0882 --- /dev/null +++ b/apps/web/src/lib/bot/images.ts @@ -0,0 +1,101 @@ +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { + CLOUD_AGENT_IMAGE_ALLOWED_TYPES, + CLOUD_AGENT_IMAGE_MAX_COUNT, + CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES, + CLOUD_AGENT_IMAGE_MIME_TO_EXTENSION, + type CloudAgentImageAllowedType, +} from '@/lib/cloud-agent/constants'; +import type { Images } from '@/lib/images-schema'; +import { r2Client, r2CloudAgentAttachmentsBucketName } from '@/lib/r2/client'; +import { captureException } from '@sentry/nextjs'; +import type { Attachment, Message } from 'chat'; +import { randomUUID } from 'crypto'; + +const ALLOWED_TYPES_SET = new Set(CLOUD_AGENT_IMAGE_ALLOWED_TYPES); + +function isAllowedImageType(mimeType: string): mimeType is CloudAgentImageAllowedType { + return ALLOWED_TYPES_SET.has(mimeType); +} + +/** + * Extract image attachments from a chat Message, download them via the + * adapter's authenticated `fetchData()`, upload to R2, and return an + * `Images` reference that can be passed to the Cloud Agent API. + * + * Returns `undefined` when the message has no usable image attachments. + */ +export async function extractAndUploadImages( + message: Message, + userId: string +): Promise { + const imageAttachments = message.attachments.filter( + (a): a is Attachment & { mimeType: string; fetchData: () => Promise } => + a.type === 'image' && + typeof a.mimeType === 'string' && + isAllowedImageType(a.mimeType) && + typeof a.fetchData === 'function' + ); + + if (imageAttachments.length === 0) return undefined; + + // Respect the Cloud Agent's per-message image limit + const toProcess = imageAttachments.slice(0, CLOUD_AGENT_IMAGE_MAX_COUNT); + + const messageUuid = randomUUID(); + const uploadResults = await Promise.allSettled( + toProcess.map(async attachment => { + const imageId = randomUUID(); + const ext = + CLOUD_AGENT_IMAGE_MIME_TO_EXTENSION[attachment.mimeType as CloudAgentImageAllowedType]; + const filename = `${imageId}.${ext}`; + const r2Key = `${userId}/cloud-agent/${messageUuid}/${filename}`; + + if ( + typeof attachment.size === 'number' && + attachment.size > CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES + ) { + throw new Error( + `Image ${attachment.name ?? filename} exceeds ${CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES / (1024 * 1024)}MB limit (${(attachment.size / (1024 * 1024)).toFixed(1)}MB)` + ); + } + + const data = await attachment.fetchData(); + + if (data.byteLength > CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES) { + throw new Error( + `Image ${attachment.name ?? filename} exceeds ${CLOUD_AGENT_IMAGE_MAX_SIZE_BYTES / (1024 * 1024)}MB limit (${(data.byteLength / (1024 * 1024)).toFixed(1)}MB)` + ); + } + + await r2Client.send( + new PutObjectCommand({ + Bucket: r2CloudAgentAttachmentsBucketName, + Key: r2Key, + Body: data, + ContentType: attachment.mimeType, + ContentLength: data.byteLength, + Metadata: { userId, messageUuid, imageId }, + }) + ); + + return filename; + }) + ); + + const filenames: string[] = []; + for (const result of uploadResults) { + if (result.status === 'fulfilled') { + filenames.push(result.value); + } else { + console.error('[KiloBot] Failed to upload image attachment:', result.reason); + captureException(result.reason, { + tags: { component: 'kilo-bot', op: 'upload-slack-image' }, + }); + } + } + + if (filenames.length === 0) return undefined; + + return { path: messageUuid, files: filenames }; +} diff --git a/apps/web/src/lib/bot/run.ts b/apps/web/src/lib/bot/run.ts index 2c3873764..bf7e57f03 100644 --- a/apps/web/src/lib/bot/run.ts +++ b/apps/web/src/lib/bot/run.ts @@ -1,8 +1,10 @@ import { updateBotRequest } from '@/lib/bot/request-logging'; import { runBotAgent } from '@/lib/bot/agent-runner'; +import { extractAndUploadImages } from '@/lib/bot/images'; import type { PlatformIntegration, User } from '@kilocode/db'; import type { Message, Thread } from 'chat'; import { emoji } from 'chat'; +import { captureException } from '@sentry/nextjs'; export async function processMessage({ thread, @@ -19,6 +21,19 @@ export async function processMessage({ }) { const startedAt = Date.now(); + // Extract and upload any image attachments from the Slack message to R2. + // This runs before the agent loop so the images are ready when a Cloud Agent + // session is spawned. Failures are non-fatal — we log and continue without images. + let images: Awaited>; + try { + images = await extractAndUploadImages(message, user.id); + } catch (error) { + console.error('[KiloBot] Failed to extract/upload images, continuing without them:', error); + captureException(error, { + tags: { component: 'kilo-bot', op: 'extract-upload-images' }, + }); + } + try { const result = await runBotAgent({ thread, @@ -28,6 +43,7 @@ export async function processMessage({ user, botRequestId, prompt: message.text, + images, }); if (botRequestId) { diff --git a/apps/web/src/lib/bot/tools/spawn-cloud-agent-session.ts b/apps/web/src/lib/bot/tools/spawn-cloud-agent-session.ts index 46d9d2705..4fd7f8e5d 100644 --- a/apps/web/src/lib/bot/tools/spawn-cloud-agent-session.ts +++ b/apps/web/src/lib/bot/tools/spawn-cloud-agent-session.ts @@ -15,6 +15,7 @@ import { getGitLabInstanceUrlForUser, buildGitLabCloneUrl, } from '@/lib/cloud-agent/gitlab-integration-helpers'; +import type { Images } from '@/lib/images-schema'; import { APP_URL } from '@/lib/constants'; import { INTERNAL_API_SECRET } from '@/lib/config.server'; import { parseBotCallbackStep } from '@/lib/bot/step-budget'; @@ -94,7 +95,7 @@ export default async function spawnCloudAgentSession( ticketUserId: string, botRequestId: string | undefined, onSessionReady?: RunSessionInput['onSessionReady'], - options?: { prSignature?: string; chatPlatform?: string; currentStep?: number } + options?: { prSignature?: string; chatPlatform?: string; currentStep?: number; images?: Images } ): Promise { console.log('[KiloBot] spawnCloudAgentSession called with args:', JSON.stringify(args, null, 2)); @@ -172,6 +173,7 @@ export default async function spawnCloudAgentSession( kilocodeOrganizationId, createdOnPlatform: chatPlatform, callbackTarget, + images: options?.images, envVars: profileConfig.envVars, encryptedSecrets: profileConfig.encryptedSecrets, setupCommands: profileConfig.setupCommands, @@ -200,6 +202,7 @@ export default async function spawnCloudAgentSession( kilocodeOrganizationId, createdOnPlatform: chatPlatform, callbackTarget, + images: options?.images, envVars: profileConfig.envVars, encryptedSecrets: profileConfig.encryptedSecrets, setupCommands: profileConfig.setupCommands,