feat: better image handling (auto resize & max size constraints)

AGENTS.md

@@ -9,6 +9,7 @@
 ### General Principles
 
 - Keep things in one function unless composable or reusable
+- Do not extract single-use helpers preemptively. Inline the logic at the call site unless the helper is reused, hides a genuinely complex boundary, or has a clear independent name that improves the caller.
 - Avoid `try`/`catch` where possible
 - Avoid using the `any` type
 - Use Bun APIs when possible, like `Bun.file()`

package.json

@@ -133,6 +133,7 @@
   },
   "patchedDependencies": {
     "@npmcli/agent@4.0.0": "patches/@npmcli%2Fagent@4.0.0.patch",
+    "@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch",
     "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch",
     "solid-js@1.9.10": "patches/solid-js@1.9.10.patch"
   }

packages/opencode/package.json

@@ -119,6 +119,7 @@
     "@opentui/solid": "catalog:",
     "@parcel/watcher": "2.5.1",
     "@pierre/diffs": "catalog:",
+    "@silvia-odwyer/photon-node": "0.3.4",
     "@solid-primitives/event-bus": "1.1.2",
     "@solid-primitives/scheduled": "1.5.2",
     "@standard-schema/spec": "1.0.0",

packages/opencode/src/audio.d.ts

@@ -2,3 +2,8 @@ declare module "*.wav" {
   const file: string
   export default file
 }
+
+declare module "*.wasm" {
+  const file: string
+  export default file
+}

packages/opencode/src/config/attachment.ts

@@ -0,0 +1,30 @@
+export * as ConfigAttachment from "./attachment"
+
+import { Schema } from "effect"
+import { zod } from "@opencode-ai/core/effect-zod"
+import { PositiveInt, withStatics } from "@opencode-ai/core/schema"
+
+export const Image = Schema.Struct({
+  auto_resize: Schema.optional(Schema.Boolean).annotate({
+    description: "Resize images before sending them to the model when they exceed configured limits (default: true)",
+  }),
+  max_width: Schema.optional(PositiveInt).annotate({
+    description: "Maximum image width before resizing or rejecting the attachment (default: 2000)",
+  }),
+  max_height: Schema.optional(PositiveInt).annotate({
+    description: "Maximum image height before resizing or rejecting the attachment (default: 2000)",
+  }),
+  max_base64_bytes: Schema.optional(PositiveInt).annotate({
+    description: "Maximum base64 payload bytes for an image attachment (default: 4718592)",
+  }),
+})
+  .annotate({ identifier: "ImageAttachmentConfig" })
+  .pipe(withStatics((s) => ({ zod: zod(s) })))
+export type Image = Schema.Schema.Type<typeof Image>
+
+export const Info = Schema.Struct({
+  image: Schema.optional(Image).annotate({ description: "Image attachment configuration" }),
+})
+  .annotate({ identifier: "AttachmentConfig" })
+  .pipe(withStatics((s) => ({ zod: zod(s) })))
+export type Info = Schema.Schema.Type<typeof Info>

packages/opencode/src/config/config.ts

@@ -25,6 +25,7 @@ import { containsPath } from "../project/instance-context"
 import { zod } from "@opencode-ai/core/effect-zod"
 import { NonNegativeInt, PositiveInt, withStatics, type DeepMutable } from "@opencode-ai/core/schema"
 import { ConfigAgent } from "./agent"
+import { ConfigAttachment } from "./attachment"
 import { ConfigCommand } from "./command"
 import { ConfigFormatter } from "./formatter"
 import { ConfigLayout } from "./layout"
@@ -241,6 +242,9 @@ export const Info = Schema.Struct({
   layout: Schema.optional(ConfigLayout.Layout).annotate({ description: "@deprecated Always uses stretch layout." }),
   permission: Schema.optional(ConfigPermission.Info),
   tools: Schema.optional(Schema.Record(Schema.String, Schema.Boolean)),
+  attachment: Schema.optional(ConfigAttachment.Info).annotate({
+    description: "Attachment processing configuration, including image size limits and resizing behavior",
+  }),
   enterprise: Schema.optional(
     Schema.Struct({
       url: Schema.optional(Schema.String).annotate({ description: "Enterprise URL" }),

packages/opencode/src/image/image.ts

@@ -0,0 +1,180 @@
+import { Config } from "@/config/config"
+import type { MessageV2 } from "@/session/message-v2"
+import * as Log from "@opencode-ai/core/util/log"
+import { Context, Effect, Layer, Schema } from "effect"
+
+const MAX_BASE64_BYTES = 4.5 * 1024 * 1024
+const MAX_WIDTH = 2000
+const MAX_HEIGHT = 2000
+const AUTO_RESIZE = true
+const JPEG_QUALITIES = [80, 85, 70, 55, 40]
+const log = Log.create({ service: "image" })
+
+export class PhotonUnavailableError extends Schema.TaggedErrorClass<PhotonUnavailableError>()(
+  "ImagePhotonUnavailableError",
+  {},
+) {
+  override get message() {
+    return "Photon image processor is unavailable"
+  }
+}
+
+export class InvalidDataUrlError extends Schema.TaggedErrorClass<InvalidDataUrlError>()("ImageInvalidDataUrlError", {
+  url: Schema.String,
+}) {
+  override get message() {
+    return "Image URL must be a base64 data URL"
+  }
+}
+
+export class DecodeError extends Schema.TaggedErrorClass<DecodeError>()("ImageDecodeError", {}) {
+  override get message() {
+    return "Image could not be decoded"
+  }
+}
+
+export class SizeError extends Schema.TaggedErrorClass<SizeError>()("ImageSizeError", {
+  bytes: Schema.Number,
+  max: Schema.Number,
+  width: Schema.Number,
+  height: Schema.Number,
+  max_width: Schema.Number,
+  max_height: Schema.Number,
+}) {
+  override get message() {
+    return `Image ${this.width}x${this.height} with base64 size ${this.bytes} exceeds configured limits and could not be resized below ${this.max_width}x${this.max_height}/${this.max} bytes`
+  }
+}
+
+export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError
+
+export interface Interface {
+  readonly normalize: (input: MessageV2.FilePart) => Effect.Effect<MessageV2.FilePart, Error>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@opencode/Image") {}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const config = yield* Config.Service
+    const loadPhoton = yield* Effect.cached(
+      Effect.promise(async () => {
+        try {
+          const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } }))
+            .default
+          // Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
+          ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
+            photonWasm
+          return await import("@silvia-odwyer/photon-node")
+        } catch {
+          return null
+        }
+      }),
+    )
+
+    const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) {
+      const image = (yield* config.get()).attachment?.image
+      const info = {
+        autoResize: image?.auto_resize ?? AUTO_RESIZE,
+        maxWidth: image?.max_width ?? MAX_WIDTH,
+        maxHeight: image?.max_height ?? MAX_HEIGHT,
+        maxBase64Bytes: image?.max_base64_bytes ?? MAX_BASE64_BYTES,
+      }
+      if (!input.url.startsWith("data:") || !input.url.includes(";base64,"))
+        return yield* new InvalidDataUrlError({ url: input.url })
+
+      const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length)
+      const photon = yield* loadPhoton
+      if (!photon) return yield* new PhotonUnavailableError()
+
+      const decoded = yield* Effect.sync(() => {
+        try {
+          return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
+        } catch {
+          return undefined
+        }
+      })
+      if (!decoded) return yield* new DecodeError()
+
+      try {
+        const originalWidth = decoded.get_width()
+        const originalHeight = decoded.get_height()
+        if (
+          originalWidth <= info.maxWidth &&
+          originalHeight <= info.maxHeight &&
+          Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes
+        )
+          return input
+        if (!info.autoResize)
+          return yield* new SizeError({
+            bytes: Buffer.byteLength(base64, "utf8"),
+            max: info.maxBase64Bytes,
+            width: originalWidth,
+            height: originalHeight,
+            max_width: info.maxWidth,
+            max_height: info.maxHeight,
+          })
+
+        const scale = Math.min(1, info.maxWidth / originalWidth, info.maxHeight / originalHeight)
+        for (const size of Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
+          const previous = acc.at(-1) ?? {
+            width: Math.max(1, Math.round(originalWidth * scale)),
+            height: Math.max(1, Math.round(originalHeight * scale)),
+          }
+          const next =
+            acc.length === 0
+              ? previous
+              : {
+                  width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
+                  height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
+                }
+          return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next]
+        }, [])) {
+          const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
+          const candidate = [
+            { data: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
+            ...JPEG_QUALITIES.map((quality) => ({
+              data: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
+              mime: "image/jpeg",
+            })),
+          ]
+            .map((item) => ({ ...item, bytes: Buffer.byteLength(item.data, "utf8") }))
+            .find((item) => item.bytes <= info.maxBase64Bytes)
+          resized.free()
+
+          if (candidate) {
+            log.info("using resized image", {
+              from_mime: input.mime,
+              to_mime: candidate.mime,
+              from: `${originalWidth}x${originalHeight}`,
+              to: `${size.width}x${size.height}`,
+            })
+            return {
+              ...input,
+              mime: candidate.mime,
+              url: `data:${candidate.mime};base64,${candidate.data}`,
+            }
+          }
+        }
+
+        return yield* new SizeError({
+          bytes: Buffer.byteLength(base64, "utf8"),
+          max: info.maxBase64Bytes,
+          width: originalWidth,
+          height: originalHeight,
+          max_width: info.maxWidth,
+          max_height: info.maxHeight,
+        })
+      } finally {
+        decoded.free()
+      }
+    })
+
+    return Service.of({ normalize })
+  }),
+)
+
+export const defaultLayer = layer.pipe(Layer.provide(Config.defaultLayer))
+
+export * as Image from "./image"

packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts

@@ -203,13 +203,15 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
       params: { sessionID: SessionID }
       payload: typeof InitPayload.Type
     }) {
-      yield* promptSvc.command({
-        sessionID: ctx.params.sessionID,
-        messageID: ctx.payload.messageID,
-        model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
-        command: Command.Default.INIT,
-        arguments: "",
-      })
+      yield* promptSvc
+        .command({
+          sessionID: ctx.params.sessionID,
+          messageID: ctx.payload.messageID,
+          model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
+          command: Command.Default.INIT,
+          arguments: "",
+        })
+        .pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
       return true
     })
 
@@ -258,18 +260,18 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
     }) {
       const instance = yield* InstanceState.context
       const workspace = yield* InstanceState.workspaceID
+      const message = yield* promptSvc
+        .prompt({
+          ...ctx.payload,
+          sessionID: ctx.params.sessionID,
+        })
+        .pipe(
+          Effect.provideService(InstanceRef, instance),
+          Effect.provideService(WorkspaceRef, workspace),
+          Effect.mapError(() => new HttpApiError.BadRequest({})),
+        )
       return HttpServerResponse.stream(
-        Stream.fromEffect(
-          promptSvc
-            .prompt({
-              ...ctx.payload,
-              sessionID: ctx.params.sessionID,
-            })
-            .pipe(Effect.provideService(InstanceRef, instance), Effect.provideService(WorkspaceRef, workspace)),
-        ).pipe(
-          Stream.map((message) => JSON.stringify(message)),
-          Stream.encodeText,
-        ),
+        Stream.make(JSON.stringify(message)).pipe(Stream.encodeText),
         { contentType: "application/json" },
       )
     })
@@ -297,7 +299,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
       params: { sessionID: SessionID }
       payload: typeof CommandPayload.Type
     }) {
-      return yield* promptSvc.command({ ...ctx.payload, sessionID: ctx.params.sessionID })
+      return yield* promptSvc
+        .command({ ...ctx.payload, sessionID: ctx.params.sessionID })
+        .pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
     })
 
     const shell = Effect.fn("SessionHttpApi.shell")(function* (ctx: {