feat(dllm): image input through the backend (multimodal C-ABI)

Routes PredictOptions.Images (raw base64, the core convention) through
dllm.cpp's probed multimodal entry points as data: URIs; the gemma4
renderer appends one engine-side <image> marker per image after the
last user message (llama.cpp attachment convention; the template's
content-parts branch is unreachable through the flattened pb shape).
The engine expands markers to boi + soft*n + eoi and splices the
vision-tower embeddings. Older libdllm.so without the mm symbols fails
with an actionable error (Dlsym probe). DLLM_VERSION pin bumped to the
engine's vision-capable commit.

Assisted-by: Claude Code (Fable 5)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

backend/go/dllm/Makefile

@@ -19,7 +19,11 @@
 # until then the anonymous clone below fails. Use the symlink shortcut above
 # with a local checkout, or a git credential helper with access to the repo.
 
-DLLM_VERSION?=b22fcebebfb225131113188599a9ae542b2935d7
+# The pin below is the first commit carrying the multimodal C-ABI entry
+# points (dllm_capi_generate_mm / dllm_capi_generate_stream_mm) the
+# image-input path probes for; older libs still load, but image requests
+# then fail with "library predates the multimodal entry points".
+DLLM_VERSION?=e6dcf44cddd65845e3a0814a1c2282a5d90ee98a
 DLLM_REPO?=https://github.com/mudler/dllm.cpp
 
 GOCMD?=go

backend/go/dllm/capi.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -45,6 +46,34 @@ var (
 	cppCancel         func(ctx uintptr)
 )
 
+// Optional multimodal entry points (dllm_capi.h's P4 surface). The ABI
+// version stays 1: presence is detected by PROBING the symbols with Dlsym at
+// boot (loadCAPI, mirroring the parakeet-cpp optional-symbol pattern). nil
+// means the loaded libdllm.so predates the mm surface; the wrappers below
+// then fail with errMMUnsupported instead of crashing on a nil call.
+var (
+	cppGenerateMM       func(ctx uintptr, prompt, imagesJSON, optsJSON string) uintptr
+	cppGenerateStreamMM func(ctx uintptr, prompt, imagesJSON, optsJSON string, onBlock, onStep, userData uintptr) int32
+)
+
+// mmImageMarker is the literal placeholder dllm_capi_generate_mm expands to
+// <boi> + soft-token placeholders + <eoi> (dllm_capi.h placeholder contract;
+// capi.cpp MM_MARKER). The prompt must carry exactly one marker per
+// images_json entry, in image order.
+const mmImageMarker = "<image>"
+
+// errMMUnsupported is returned for image-bearing requests against an old
+// text-only libdllm.so (the Dlsym probe found no mm symbols).
+var errMMUnsupported = errors.New(
+	"dllm: image input requires libdllm.so with the multimodal entry points (dllm_capi_generate_mm), but the loaded library predates them - rebuild/upgrade the dllm backend to use images")
+
+// cMMSupported reports whether the loaded libdllm.so carries the multimodal
+// generate pair. Both symbols ship together (same dllm.cpp commit), but the
+// guard requires both anyway so a half-present surface can never dispatch.
+func cMMSupported() bool {
+	return cppGenerateMM != nil && cppGenerateStreamMM != nil
+}
+
 // cAbiVersion returns the library's DLLM_CAPI_ABI_VERSION.
 func cAbiVersion() int32 {
 	return cppAbiVersion()
@@ -108,6 +137,23 @@ func cGenerate(h uintptr, prompt, optsJSON string) (string, error) {
 	return out, nil
 }
 
+// cGenerateMM is cGenerate's multimodal counterpart. imagesJSON is the flat
+// JSON array of image entries (data: base64 URIs here; the C side also takes
+// file paths) and the prompt must carry one mmImageMarker per entry - the
+// engine enforces the 1:1 match and reports mismatches through last_error.
+func cGenerateMM(h uintptr, prompt, imagesJSON, optsJSON string) (string, error) {
+	if !cMMSupported() {
+		return "", errMMUnsupported
+	}
+	ret := cppGenerateMM(h, prompt, imagesJSON, optsJSON)
+	if ret == 0 {
+		return "", fmt.Errorf("dllm: generate_mm failed: %s", lastErrorOr(h, "unknown error"))
+	}
+	out := goStringFromCPtr(ret)
+	cppFreeString(ret)
+	return out, nil
+}
+
 // streamCallState carries the Go callbacks for one in-flight
 // cGenerateStream call; the registry key travels through C as user_data.
 // The map shape mirrors the whisper backend's streamCallStates: only one
@@ -158,11 +204,12 @@ func onStepTrampoline(step int32, totalSteps int32, canvasPreview uintptr, userD
 	}
 }
 
-// cGenerateStream runs a generation with per-committed-block (onBlock) and
-// per-denoising-step (onStep) callbacks; either may be nil. The callbacks
-// run on the C thread (see the trampoline docs). Returns an error carrying
-// last_error on failure; cancellation surfaces as the "cancelled" message.
-func cGenerateStream(h uintptr, prompt, optsJSON string, onBlock func(text string), onStep func(step, total int, preview string)) error {
+// withStreamCallbacks registers onBlock/onStep in the trampoline registry
+// for the duration of one streaming C call and invokes call with the C
+// function pointers (NULL for absent callbacks, so the C side skips the
+// per-block / per-step detokenize work entirely) plus the registry key to
+// pass as user_data. Shared by the text and multimodal stream wrappers.
+func withStreamCallbacks(onBlock func(text string), onStep func(step, total int, preview string), call func(blockPtr, stepPtr, userData uintptr) int32) int32 {
 	streamCbOnce.Do(func() {
 		blockCbPtr = purego.NewCallback(onBlockTrampoline)
 		stepCbPtr = purego.NewCallback(onStepTrampoline)
@@ -172,22 +219,45 @@ func cGenerateStream(h uintptr, prompt, optsJSON string, onBlock func(text strin
 	streamCallStates.Store(id, &streamCallState{onBlock: onBlock, onStep: onStep})
 	defer streamCallStates.Delete(id)
 
-	// Pass NULL for absent callbacks so the C side skips the per-block /
-	// per-step detokenize work entirely.
 	var blockPtr, stepPtr uintptr
 	if onBlock != nil {
 		blockPtr = blockCbPtr
 	}
 	if onStep != nil {
 		stepPtr = stepCbPtr
 	}
+	return call(blockPtr, stepPtr, uintptr(id))
+}
 
-	if rc := cppGenerateStream(h, prompt, optsJSON, blockPtr, stepPtr, uintptr(id)); rc != 0 {
+// cGenerateStream runs a generation with per-committed-block (onBlock) and
+// per-denoising-step (onStep) callbacks; either may be nil. The callbacks
+// run on the C thread (see the trampoline docs). Returns an error carrying
+// last_error on failure; cancellation surfaces as the "cancelled" message.
+func cGenerateStream(h uintptr, prompt, optsJSON string, onBlock func(text string), onStep func(step, total int, preview string)) error {
+	rc := withStreamCallbacks(onBlock, onStep, func(blockPtr, stepPtr, userData uintptr) int32 {
+		return cppGenerateStream(h, prompt, optsJSON, blockPtr, stepPtr, userData)
+	})
+	if rc != 0 {
 		return fmt.Errorf("dllm: generate_stream failed: %s", lastErrorOr(h, "unknown error"))
 	}
 	return nil
 }
 
+// cGenerateStreamMM is cGenerateStream's multimodal counterpart; see
+// cGenerateMM for the imagesJSON/marker contract.
+func cGenerateStreamMM(h uintptr, prompt, imagesJSON, optsJSON string, onBlock func(text string), onStep func(step, total int, preview string)) error {
+	if !cMMSupported() {
+		return errMMUnsupported
+	}
+	rc := withStreamCallbacks(onBlock, onStep, func(blockPtr, stepPtr, userData uintptr) int32 {
+		return cppGenerateStreamMM(h, prompt, imagesJSON, optsJSON, blockPtr, stepPtr, userData)
+	})
+	if rc != 0 {
+		return fmt.Errorf("dllm: generate_stream_mm failed: %s", lastErrorOr(h, "unknown error"))
+	}
+	return nil
+}
+
 // cCancel requests cancellation of the in-flight generate on h. This is the
 // ONE entry point safe to call from any goroutine while a generate runs (it
 // only flips an atomic). Note the cancel-reset race from the header: each

backend/go/dllm/dllm.go

@@ -42,6 +42,13 @@ type generator interface {
 	// generateStream invokes onBlock once per committed diffusion block, on
 	// the thread running the C call, before returning.
 	generateStream(prompt, optsJSON string, onBlock func(text string)) error
+	// generateMM / generateStreamMM are the multimodal counterparts:
+	// imagesJSON is a flat JSON array of data: base64 URIs and the prompt
+	// carries one mmImageMarker per entry (dllm_capi.h placeholder
+	// contract). Against an old text-only libdllm.so they fail with
+	// errMMUnsupported.
+	generateMM(prompt, imagesJSON, optsJSON string) (string, error)
+	generateStreamMM(prompt, imagesJSON, optsJSON string, onBlock func(text string)) error
 	tokenizeJSON(text string) (string, error)
 	// cancel is the ONE entry point safe to call concurrently with an
 	// in-flight generate on the same ctx (dllm_capi.h: it only flips an
@@ -66,6 +73,15 @@ func (g *capiGenerator) generateStream(prompt, optsJSON string, onBlock func(tex
 	return cGenerateStream(g.h, prompt, optsJSON, onBlock, nil)
 }
 
+func (g *capiGenerator) generateMM(prompt, imagesJSON, optsJSON string) (string, error) {
+	return cGenerateMM(g.h, prompt, imagesJSON, optsJSON)
+}
+
+func (g *capiGenerator) generateStreamMM(prompt, imagesJSON, optsJSON string, onBlock func(text string)) error {
+	// on_step is nil for the same reason as generateStream.
+	return cGenerateStreamMM(g.h, prompt, imagesJSON, optsJSON, onBlock, nil)
+}
+
 func (g *capiGenerator) tokenizeJSON(text string) (string, error) {
 	return cTokenizeJSON(g.h, text)
 }
@@ -267,20 +283,55 @@ func metadataEnableThinking(opts *pb.PredictOptions) bool {
 }
 
 // buildPrompt resolves the prompt for a request. With use_tokenizer_template
-// and raw messages the backend owns templating (RenderGemma4) and the output
-// is in the known gemma4 format, so parse=true. Without it the caller
-// templated the prompt themselves (LocalAI's Go templates + PEG fallback, or
-// a bare completion): the prompt passes through verbatim and the output is
-// NOT gemma4-parsed - it is emitted as plain content and the Go side's
-// extraction applies, as for any non-autoparsing backend.
+// and raw messages the backend owns templating (RenderGemma4, including the
+// mmImageMarker injection for opts.Images) and the output is in the known
+// gemma4 format, so parse=true. Without it the caller templated the prompt
+// themselves (LocalAI's Go templates + PEG fallback, or a bare completion):
+// the prompt passes through verbatim - for image requests it must already
+// carry one literal mmImageMarker per image (the engine enforces the 1:1
+// match) - and the output is NOT gemma4-parsed - it is emitted as plain
+// content and the Go side's extraction applies, as for any non-autoparsing
+// backend.
 func buildPrompt(opts *pb.PredictOptions) (prompt string, parse bool, err error) {
 	if opts.GetUseTokenizerTemplate() && len(opts.GetMessages()) > 0 {
-		prompt, err = RenderGemma4(opts.GetMessages(), opts.GetTools(), metadataEnableThinking(opts), true)
+		prompt, err = RenderGemma4(opts.GetMessages(), opts.GetTools(), len(opts.GetImages()), metadataEnableThinking(opts), true)
 		return prompt, true, err
 	}
 	return opts.GetPrompt(), false, nil
 }
 
+// imagesJSON renders opts.Images as the flat JSON array of data: URIs the mm
+// C-ABI expects, or "" when the request carries no images. The entries arrive
+// as RAW base64 payloads: LocalAI's OpenAI layer decodes every image_url /
+// image content part (URL download or data: URI) to plain base64 via
+// utils.GetContentURIAsBase64 (core/http/middleware/request.go) and core
+// flattens them into PredictOptions.Images (core/backend/llm.go). The
+// hardcoded image/jpeg mime mirrors the llama.cpp backend's re-wrapping
+// convention (grpc-server.cpp, "data:image/jpeg;base64," + images(i)); the
+// engine ignores the declared mime and sniffs the real format from the
+// decoded bytes (stb_image), so PNG/BMP payloads work through it too.
+func imagesJSON(images []string) (string, error) {
+	if len(images) == 0 {
+		return "", nil
+	}
+	uris := make([]string, len(images))
+	for i, img := range images {
+		// dllm_capi.h: array entries are read VERBATIM up to the closing
+		// quote, with NO escape handling. json.Marshal would escape these
+		// bytes and the C side would misparse the entry, so fail loud (they
+		// can never appear in genuine base64 anyway).
+		if strings.ContainsAny(img, "\"\\") {
+			return "", fmt.Errorf("dllm: image %d is not base64 (contains a quote or backslash; PredictOptions.Images entries must be raw base64 payloads)", i)
+		}
+		uris[i] = "data:image/jpeg;base64," + img
+	}
+	b, err := json.Marshal(uris)
+	if err != nil {
+		return "", fmt.Errorf("dllm: marshal images: %w", err)
+	}
+	return string(b), nil
+}
+
 // requestOptsJSON merges the model-level overrides with the request's
 // sampling fields into the flat opts JSON for one generate call.
 func (d *Dllm) requestOptsJSON(opts *pb.PredictOptions) (string, error) {
@@ -307,17 +358,27 @@ func (d *Dllm) requestOptsJSON(opts *pb.PredictOptions) (string, error) {
 
 // prepareRequest is the shared prologue of the rich methods: resolve the
 // prompt (and whether the output gets gemma4-parsed) and build the per-call
-// opts JSON.
-func (d *Dllm) prepareRequest(opts *pb.PredictOptions) (prompt string, parse bool, optsJSON string, err error) {
+// opts JSON plus the images JSON ("" for text-only requests, which routes
+// the call through the text generate entry points).
+func (d *Dllm) prepareRequest(opts *pb.PredictOptions) (prompt string, parse bool, optsJSON, imgJSON string, err error) {
+	// Fail loud on media the engine has no path for, instead of silently
+	// generating from a prompt that ignores them.
+	if len(opts.GetVideos()) > 0 || len(opts.GetAudios()) > 0 {
+		return "", false, "", "", errors.New("dllm: video/audio input is not supported (images only)")
+	}
 	prompt, parse, err = buildPrompt(opts)
 	if err != nil {
-		return "", false, "", err
+		return "", false, "", "", err
 	}
 	optsJSON, err = d.requestOptsJSON(opts)
 	if err != nil {
-		return "", false, "", err
+		return "", false, "", "", err
 	}
-	return prompt, parse, optsJSON, nil
+	imgJSON, err = imagesJSON(opts.GetImages())
+	if err != nil {
+		return "", false, "", "", err
+	}
+	return prompt, parse, optsJSON, imgJSON, nil
 }
 
 // sanitizeUTF8 makes s safe for a proto3 string field. Block-boundary
@@ -386,15 +447,19 @@ func (d *Dllm) PredictRich(opts *pb.PredictOptions) (*pb.Reply, error) {
 	if d.gen == nil {
 		return nil, grpcerrors.ModelNotLoaded("dllm")
 	}
-	prompt, parse, optsJSON, err := d.prepareRequest(opts)
+	prompt, parse, optsJSON, imgJSON, err := d.prepareRequest(opts)
 	if err != nil {
 		return nil, err
 	}
 
 	var out string
 	var genErr error
 	d.submit(func() {
-		out, genErr = d.gen.generate(prompt, optsJSON)
+		if imgJSON != "" {
+			out, genErr = d.gen.generateMM(prompt, imgJSON, optsJSON)
+		} else {
+			out, genErr = d.gen.generate(prompt, optsJSON)
+		}
 	})
 	if genErr != nil {
 		return nil, genErr
@@ -429,7 +494,7 @@ func (d *Dllm) PredictStreamRich(opts *pb.PredictOptions, results chan<- *pb.Rep
 	if d.gen == nil {
 		return grpcerrors.ModelNotLoaded("dllm")
 	}
-	prompt, parse, optsJSON, err := d.prepareRequest(opts)
+	prompt, parse, optsJSON, imgJSON, err := d.prepareRequest(opts)
 	if err != nil {
 		return err
 	}
@@ -467,7 +532,11 @@ func (d *Dllm) PredictStreamRich(opts *pb.PredictOptions, results chan<- *pb.Rep
 
 	var genErr error
 	d.submit(func() {
-		genErr = d.gen.generateStream(prompt, optsJSON, onBlock)
+		if imgJSON != "" {
+			genErr = d.gen.generateStreamMM(prompt, imgJSON, optsJSON, onBlock)
+		} else {
+			genErr = d.gen.generateStream(prompt, optsJSON, onBlock)
+		}
 	})
 	if genErr != nil {
 		return genErr