fix(vision): probe the media marker for pinned llama.cpp backend variants (#10955)

llama.cpp picks a random per-process media marker (ggml-org/llama.cpp#21962),
so LocalAI renders the prompt with a "<__media__>" sentinel and swaps in the
backend's real marker after probing ModelMetadata.

That probe was gated on an exact match against "llama-cpp", the gallery's meta
backend name. A model config pinning a concrete build ("vulkan-llama-cpp",
"cuda12-llama-cpp", "rocm-llama-cpp", ... and their -development counterparts)
runs the same llama.cpp gRPC server but skipped the probe, so MediaMarker
stayed empty, no substitution happened, and the prompt reached mtmd still
carrying the sentinel. mtmd_tokenize then counted zero markers against one
bitmap and every image request failed with "Failed to tokenize prompt".

The same early return also skipped thinking-mode detection and tool-format
marker extraction, so a pinned variant silently lost reasoning and native
tool-call parsing too.

Add IsLlamaCppBackend, which recognises the whole variant family (plus the
empty auto-detect name, which resolves to llama.cpp) while excluding
ik-llama.cpp, a separate engine that merely shares the suffix.

Fixes #10945


Assisted-by: Claude Code:claude-opus-4-8[1m] [Read] [Edit] [Bash]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

Author: localai-bot

core/config/backend_capabilities.go

@@ -621,6 +621,40 @@ func NormalizeBackendName(backend string) string {
 	return strings.ReplaceAll(backend, ".", "-")
 }
 
+// llamaCppChannelSuffixes are the release-channel suffixes appended to a
+// llama.cpp backend name in the gallery ("llama-cpp" vs
+// "llama-cpp-development"). They carry no engine information, so they are
+// stripped before the family check below.
+var llamaCppChannelSuffixes = []string{"-development", "-quantization"}
+
+// IsLlamaCppBackend reports whether a backend name refers to a build of the
+// llama.cpp gRPC server. The gallery ships one concrete backend per hardware
+// capability ("vulkan-llama-cpp", "cuda12-llama-cpp", "metal-llama-cpp", ...)
+// behind the "llama-cpp" meta name, and an operator may pin any of them in a
+// model config. They all run the same server, so anything gated on "is this
+// llama.cpp" must accept the whole family: an exact match against "llama-cpp"
+// silently skips every pinned variant (see #10945, where skipping the media
+// marker probe broke all vision requests).
+//
+// The empty name matches too: it is the GGUF auto-detect path, which resolves
+// to llama.cpp.
+//
+// ik-llama.cpp is deliberately excluded. It is a separate engine with its own
+// gRPC server that happens to share the "-llama-cpp" suffix.
+func IsLlamaCppBackend(backend string) bool {
+	name := NormalizeBackendName(backend)
+	if name == "" {
+		return true
+	}
+	for _, suffix := range llamaCppChannelSuffixes {
+		name = strings.TrimSuffix(name, suffix)
+	}
+	if strings.HasSuffix(name, "ik-llama-cpp") {
+		return false
+	}
+	return name == "llama-cpp" || strings.HasSuffix(name, "-llama-cpp")
+}
+
 // nonLlamaSamplerBackends lists backends whose native sampler defaults differ
 // from llama.cpp's, so LocalAI must NOT inject llama.cpp's top_k=40 default for
 // them (issue #6632). mlx_lm's intended default is top_k=0 (disabled) and mlx

core/config/backend_capabilities_test.go

@@ -109,6 +109,39 @@ var _ = Describe("IsValidUsecaseForBackend", func() {
 	})
 })
 
+var _ = Describe("IsLlamaCppBackend", func() {
+	DescribeTable("classifies a backend name",
+		func(backend string, expected bool) {
+			Expect(IsLlamaCppBackend(backend)).To(Equal(expected))
+		},
+		Entry("meta name", "llama-cpp", true),
+		Entry("dotted spelling", "llama.cpp", true),
+		Entry("auto-detect (empty)", "", true),
+		Entry("development channel", "llama-cpp-development", true),
+		Entry("quantization channel", "llama-cpp-quantization", true),
+		Entry("vulkan variant", "vulkan-llama-cpp", true),
+		Entry("cuda 12 variant", "cuda12-llama-cpp", true),
+		Entry("cuda 13 variant", "cuda13-llama-cpp", true),
+		Entry("jetson variant", "cuda13-nvidia-l4t-arm64-llama-cpp", true),
+		Entry("rocm variant", "rocm-llama-cpp", true),
+		Entry("metal variant", "metal-llama-cpp", true),
+		Entry("intel sycl f16 variant", "intel-sycl-f16-llama-cpp", true),
+		Entry("intel sycl f32 variant", "intel-sycl-f32-llama-cpp", true),
+		Entry("cpu variant", "cpu-llama-cpp", true),
+		Entry("variant on the development channel", "rocm-llama-cpp-development", true),
+		Entry("darwin quantization variant", "metal-darwin-arm64-llama-cpp-quantization", true),
+		// ik-llama.cpp is a distinct engine that merely shares the suffix.
+		Entry("ik-llama-cpp", "ik-llama-cpp", false),
+		Entry("ik-llama-cpp development", "ik-llama-cpp-development", false),
+		Entry("cpu ik-llama-cpp", "cpu-ik-llama-cpp", false),
+		Entry("cpu ik-llama-cpp development", "cpu-ik-llama-cpp-development", false),
+		Entry("vllm", "vllm", false),
+		Entry("mlx", "mlx", false),
+		Entry("whisper", "whisper", false),
+		Entry("bark-cpp", "bark-cpp", false),
+	)
+})
+
 var _ = Describe("AllBackendNames", func() {
 	It("returns 30+ backends in sorted order", func() {
 		names := AllBackendNames()

core/config/gguf.go

@@ -142,7 +142,10 @@ func DetectThinkingSupportFromBackend(ctx context.Context, cfg *ModelConfig, bac
 
 	// Only llama-cpp exposes ModelMetadata today. Other backends will either error
 	// or return an empty response — both are fine, we just bail before calling.
-	if cfg.Backend != "llama-cpp" {
+	// The check must cover every llama.cpp build, not just the "llama-cpp" meta
+	// name: a config pinning a concrete variant ("vulkan-llama-cpp", ...) runs the
+	// same server and needs the same probe (#10945).
+	if !IsLlamaCppBackend(cfg.Backend) {
 		xlog.Debug("[gguf] DetectThinkingSupportFromBackend: skipping detection", "backend", cfg.Backend)
 		return
 	}

core/config/gguf_media_marker_test.go

@@ -0,0 +1,82 @@
+package config
+
+import (
+	"context"
+
+	"github.com/mudler/LocalAI/pkg/grpc"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	grpclib "google.golang.org/grpc"
+)
+
+// markerBackend answers ModelMetadata with a fixed marker and records whether it
+// was called at all. Embedding grpc.Backend keeps the rest of the (large)
+// interface unimplemented on purpose: DetectThinkingSupportFromBackend must only
+// reach for ModelMetadata, and any other call would panic loudly.
+type markerBackend struct {
+	grpc.Backend
+	marker string
+	called bool
+}
+
+func (m *markerBackend) ModelMetadata(ctx context.Context, in *pb.ModelOptions, opts ...grpclib.CallOption) (*pb.ModelMetadataResponse, error) {
+	m.called = true
+	return &pb.ModelMetadataResponse{MediaMarker: m.marker}, nil
+}
+
+var _ = Describe("media marker probing across llama.cpp backend variants", func() {
+	// llama.cpp picks a random per-process media marker (ggml-org/llama.cpp#21962),
+	// so the marker MUST come from the backend. A config that pins a concrete
+	// variant name still runs the same llama.cpp gRPC server and must be probed,
+	// or the rendered prompt keeps LocalAI's "<__media__>" sentinel and
+	// mtmd_tokenize reports zero markers against one bitmap (#10945).
+	const backendMarker = "<__media_9fJqQpVb2hZK3nT7__>"
+
+	probe := func(backend string) *ModelConfig {
+		cfg := &ModelConfig{}
+		cfg.Backend = backend
+		client := &markerBackend{marker: backendMarker}
+		DetectThinkingSupportFromBackend(context.Background(), cfg, client, &pb.ModelOptions{})
+		return cfg
+	}
+
+	DescribeTable("captures the backend-reported marker",
+		func(backend string) {
+			Expect(probe(backend).MediaMarker).To(Equal(backendMarker))
+		},
+		Entry("meta backend", "llama-cpp"),
+		Entry("auto-detected (empty) backend", ""),
+		Entry("development channel", "llama-cpp-development"),
+		Entry("vulkan variant", "vulkan-llama-cpp"),
+		Entry("vulkan development variant", "vulkan-llama-cpp-development"),
+		Entry("cuda 12 variant", "cuda12-llama-cpp"),
+		Entry("cuda 13 jetson variant", "cuda13-nvidia-l4t-arm64-llama-cpp"),
+		Entry("rocm variant", "rocm-llama-cpp"),
+		Entry("metal variant", "metal-llama-cpp"),
+		Entry("intel sycl variant", "intel-sycl-f16-llama-cpp"),
+		Entry("cpu variant", "cpu-llama-cpp"),
+		Entry("quantization variant", "llama-cpp-quantization"),
+	)
+
+	DescribeTable("leaves the marker untouched for backends that are not llama.cpp",
+		func(backend string) {
+			cfg := &ModelConfig{}
+			cfg.Backend = backend
+			client := &markerBackend{marker: backendMarker}
+			DetectThinkingSupportFromBackend(context.Background(), cfg, client, &pb.ModelOptions{})
+			Expect(client.called).To(BeFalse(), "backend %q must not be probed", backend)
+			Expect(cfg.MediaMarker).To(BeEmpty())
+		},
+		// ik-llama.cpp is a separate engine with its own gRPC server that still
+		// uses mtmd_default_marker(), so the sentinel already matches and there
+		// is nothing to probe.
+		Entry("ik-llama-cpp", "ik-llama-cpp"),
+		Entry("ik-llama-cpp development", "ik-llama-cpp-development"),
+		Entry("cpu ik-llama-cpp", "cpu-ik-llama-cpp"),
+		Entry("vllm", "vllm"),
+		Entry("mlx", "mlx"),
+		Entry("whisper", "whisper"),
+	)
+})