feat(ollama): report model capabilities + details on /api/tags and /api/show

core/http/endpoints/ollama/capabilities.go

@@ -0,0 +1,142 @@
+package ollama
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/config"
+)
+
+// modelCapabilities maps a LocalAI ModelConfig to the Ollama capability strings
+// (https://github.com/ollama/ollama/blob/main/docs/api.md#show-model-information).
+//
+// Ollama clients use these to decide which models are eligible for a given task
+// (e.g. only allow embedding models in an "embedding model" picker). Returning
+// an empty list makes clients assume "completion" everywhere, which is wrong
+// for embedding/rerank/audio backends — see issue #9760.
+func modelCapabilities(cfg *config.ModelConfig) []string {
+	if cfg == nil {
+		return nil
+	}
+
+	var caps []string
+
+	if cfg.HasUsecases(config.FLAG_EMBEDDINGS) {
+		caps = append(caps, "embedding")
+	}
+
+	chatCapable := cfg.HasUsecases(config.FLAG_CHAT) || cfg.HasUsecases(config.FLAG_COMPLETION)
+	if chatCapable {
+		caps = append(caps, "completion")
+	}
+
+	if chatCapable && hasVisionSupport(cfg) {
+		caps = append(caps, "vision")
+	}
+
+	if chatCapable && hasToolSupport(cfg) {
+		caps = append(caps, "tools")
+	}
+
+	if chatCapable && hasThinkingSupport(cfg) {
+		caps = append(caps, "thinking")
+	}
+
+	if chatCapable && cfg.TemplateConfig.Completion != "" {
+		caps = append(caps, "insert")
+	}
+
+	return caps
+}
+
+// hasVisionSupport reports whether the model can accept image inputs. We avoid
+// cfg.HasUsecases(FLAG_VISION) because GuessUsecases has no FLAG_VISION case
+// and returns true for any chat model — see core/config/model_config.go. Instead
+// we look for explicit signals: KnownUsecases bit, multimodal projector, or
+// template/backend-reported multimodal markers.
+func hasVisionSupport(cfg *config.ModelConfig) bool {
+	if cfg.KnownUsecases != nil && (*cfg.KnownUsecases&config.FLAG_VISION) == config.FLAG_VISION {
+		return true
+	}
+	if cfg.MMProj != "" {
+		return true
+	}
+	if cfg.TemplateConfig.Multimodal != "" {
+		return true
+	}
+	if cfg.MediaMarker != "" {
+		return true
+	}
+	return false
+}
+
+// hasToolSupport reports whether the model is wired up for tool / function calling.
+// We look for any of the explicit configuration knobs LocalAI uses to drive
+// function-call extraction (regex match, response regex, grammar triggers, XML
+// format) or for the auto-detected tool-format markers populated by the
+// llama.cpp backend during model load.
+func hasToolSupport(cfg *config.ModelConfig) bool {
+	fc := cfg.FunctionsConfig
+	if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
+		return true
+	}
+	if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
+		return true
+	}
+	if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
+		return true
+	}
+	if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
+		return true
+	}
+	return false
+}
+
+// hasThinkingSupport reports whether the model has reasoning / thinking enabled.
+// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
+// when the backend probe reports that the model supports thinking.
+func hasThinkingSupport(cfg *config.ModelConfig) bool {
+	rc := cfg.ReasoningConfig
+	if rc.DisableReasoning != nil && !*rc.DisableReasoning {
+		return true
+	}
+	if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
+		// Explicit thinking markers imply support unless explicitly disabled.
+		return rc.DisableReasoning == nil || !*rc.DisableReasoning
+	}
+	return false
+}
+
+// quantRegex matches GGUF-style quantization suffixes (Q4_K_M, Q8_0, IQ3_XS, F16, ...).
+// Matches the convention used by GGUF tooling and what ggml-org/llama.cpp report.
+var quantRegex = regexp.MustCompile(`(?i)(IQ\d+(?:_[A-Z0-9]+)*|Q\d+(?:_[A-Z0-9]+)*|F16|F32|BF16)`)
+
+// paramSizeRegex matches a parameter-size token surrounded by separators
+// (e.g. "-7B-", "_3b.", ".70B-"). Avoids matching the "7" inside "Qwen3".
+var paramSizeRegex = regexp.MustCompile(`(?i)(?:^|[-_.])(\d+(?:\.\d+)?[BM])(?:[-_.]|$)`)
+
+// extractQuantizationLevel pulls the quantization tag from the model filename.
+// Returns the uppercased token (e.g. "Q4_K_M") or "" when not present.
+func extractQuantizationLevel(modelFile string) string {
+	if modelFile == "" {
+		return ""
+	}
+	base := strings.TrimSuffix(modelFile, ".gguf")
+	if m := quantRegex.FindString(base); m != "" {
+		return strings.ToUpper(m)
+	}
+	return ""
+}
+
+// extractParameterSize pulls the parameter count from the model filename.
+// Returns "" when no recognizable token is present.
+func extractParameterSize(modelFile string) string {
+	if modelFile == "" {
+		return ""
+	}
+	base := strings.TrimSuffix(modelFile, ".gguf")
+	if m := paramSizeRegex.FindStringSubmatch(base); len(m) > 1 {
+		return strings.ToUpper(m[1])
+	}
+	return ""
+}

core/http/endpoints/ollama/capabilities_test.go

@@ -0,0 +1,138 @@
+package ollama
+
+import (
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/functions"
+	"github.com/mudler/LocalAI/pkg/reasoning"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func boolPtr(b bool) *bool { return &b }
+
+func withKnownUsecases(cfg config.ModelConfig, flags ...string) config.ModelConfig {
+	cfg.KnownUsecaseStrings = flags
+	cfg.KnownUsecases = config.GetUsecasesFromYAML(flags)
+	return cfg
+}
+
+var _ = Describe("modelCapabilities", func() {
+	DescribeTable("derives Ollama capability strings from a ModelConfig",
+		func(cfg config.ModelConfig, expected []string) {
+			caps := modelCapabilities(&cfg)
+			if len(expected) == 0 {
+				Expect(caps).To(BeEmpty())
+				return
+			}
+			Expect(caps).To(ConsistOf(expected))
+		},
+		Entry("an embedding-only model exposes the embedding capability",
+			config.ModelConfig{
+				Name:       "embed-model",
+				Backend:    "llama-cpp",
+				Embeddings: boolPtr(true),
+			},
+			[]string{"embedding"},
+		),
+		Entry("a chat-template model exposes the completion capability",
+			config.ModelConfig{
+				Name:    "chat-model",
+				Backend: "llama-cpp",
+				TemplateConfig: config.TemplateConfig{
+					Chat: "{{ .Input }}",
+				},
+			},
+			[]string{"completion"},
+		),
+		Entry("a vision-capable chat model exposes completion + vision",
+			withKnownUsecases(config.ModelConfig{
+				Name:    "vision-model",
+				Backend: "llama-cpp",
+				TemplateConfig: config.TemplateConfig{
+					Chat:       "{{ .Input }}",
+					Multimodal: "<__media__>",
+				},
+			}, "FLAG_CHAT", "FLAG_VISION"),
+			[]string{"completion", "vision"},
+		),
+		Entry("a model with reasoning enabled exposes the thinking capability",
+			config.ModelConfig{
+				Name:    "thinking-model",
+				Backend: "llama-cpp",
+				TemplateConfig: config.TemplateConfig{
+					Chat: "{{ .Input }}",
+				},
+				ReasoningConfig: reasoning.Config{
+					DisableReasoning: boolPtr(false),
+				},
+			},
+			[]string{"completion", "thinking"},
+		),
+		Entry("a model with detected tool-format markers exposes the tools capability",
+			config.ModelConfig{
+				Name:    "tools-model",
+				Backend: "llama-cpp",
+				TemplateConfig: config.TemplateConfig{
+					Chat: "{{ .Input }}",
+				},
+				FunctionsConfig: functions.FunctionsConfig{
+					ToolFormatMarkers: &functions.ToolFormatMarkers{FormatType: "json_native"},
+				},
+			},
+			[]string{"completion", "tools"},
+		),
+		Entry("a model with an explicit JSON regex match exposes the tools capability",
+			config.ModelConfig{
+				Name:    "tools-regex-model",
+				Backend: "llama-cpp",
+				TemplateConfig: config.TemplateConfig{
+					Chat: "{{ .Input }}",
+				},
+				FunctionsConfig: functions.FunctionsConfig{
+					JSONRegexMatch: []string{`(?s).*`},
+				},
+			},
+			[]string{"completion", "tools"},
+		),
+		Entry("a pure backend-only model (no template, no embeddings) reports no capabilities",
+			config.ModelConfig{
+				Name:    "rerank-model",
+				Backend: "rerankers",
+			},
+			[]string{},
+		),
+	)
+})
+
+var _ = Describe("modelDetailsFromModelConfig", func() {
+	It("reports gguf format and llama-cpp family/families for a llama-cpp model", func() {
+		cfg := config.ModelConfig{
+			Name:    "llama",
+			Backend: "llama-cpp",
+		}
+		details := modelDetailsFromModelConfig(&cfg)
+		Expect(details.Format).To(Equal("gguf"))
+		Expect(details.Family).To(Equal("llama-cpp"))
+		Expect(details.Families).To(ConsistOf("llama-cpp"))
+	})
+
+	It("extracts quantization_level from the model filename when present", func() {
+		cfg := config.ModelConfig{
+			Name:    "qwen-q4",
+			Backend: "llama-cpp",
+		}
+		cfg.Model = "Qwen3-4B-Instruct-Q4_K_M.gguf"
+		details := modelDetailsFromModelConfig(&cfg)
+		Expect(details.QuantizationLevel).To(Equal("Q4_K_M"))
+	})
+
+	It("extracts parameter_size from the model filename when present", func() {
+		cfg := config.ModelConfig{
+			Name:    "qwen-4b",
+			Backend: "llama-cpp",
+		}
+		cfg.Model = "Qwen3-4B-Instruct-Q4_K_M.gguf"
+		details := modelDetailsFromModelConfig(&cfg)
+		Expect(details.ParameterSize).To(Equal("4B"))
+	})
+})

core/http/endpoints/ollama/models.go

@@ -32,13 +32,15 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader) ec
 
 			digest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name)))
 
+			details, caps := modelMetaFromConfig(bcl, name)
 			entry := schema.OllamaModelEntry{
-				Name:       ollamaName,
-				Model:      ollamaName,
-				ModifiedAt: time.Now().UTC(),
-				Size:       0,
-				Digest:     digest,
-				Details:    modelDetailsFromConfig(bcl, name),
+				Name:         ollamaName,
+				Model:        ollamaName,
+				ModifiedAt:   time.Now().UTC(),
+				Size:         0,
+				Digest:       digest,
+				Details:      details,
+				Capabilities: caps,
 			}
 			models = append(models, entry)
 		}
@@ -72,10 +74,12 @@ func ShowModelEndpoint(bcl *config.ModelConfigLoader) echo.HandlerFunc {
 		}
 
 		resp := schema.OllamaShowResponse{
-			Modelfile:  fmt.Sprintf("FROM %s", cfg.Model),
-			Parameters: "",
-			Template:   cfg.TemplateConfig.Chat,
-			Details:    modelDetailsFromModelConfig(&cfg),
+			Modelfile:    fmt.Sprintf("FROM %s", cfg.Model),
+			Parameters:   "",
+			Template:     cfg.TemplateConfig.Chat,
+			Details:      modelDetailsFromModelConfig(&cfg),
+			ModelInfo:    modelInfoFromModelConfig(&cfg),
+			Capabilities: modelCapabilities(&cfg),
 		}
 
 		return c.JSON(200, resp)
@@ -95,14 +99,16 @@ func ListRunningEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader) e
 				ollamaName += ":latest"
 			}
 
+			details, caps := modelMetaFromConfig(bcl, name)
 			entry := schema.OllamaPsEntry{
-				Name:      ollamaName,
-				Model:     ollamaName,
-				Size:      0,
-				Digest:    fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name))),
-				Details:   modelDetailsFromConfig(bcl, name),
-				ExpiresAt: time.Now().Add(24 * time.Hour).UTC(),
-				SizeVRAM:  0,
+				Name:         ollamaName,
+				Model:        ollamaName,
+				Size:         0,
+				Digest:       fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name))),
+				Details:      details,
+				ExpiresAt:    time.Now().Add(24 * time.Hour).UTC(),
+				SizeVRAM:     0,
+				Capabilities: caps,
 			}
 			models = append(models, entry)
 		}
@@ -125,18 +131,46 @@ func HeartbeatEndpoint() echo.HandlerFunc {
 	}
 }
 
-func modelDetailsFromConfig(bcl *config.ModelConfigLoader, name string) schema.OllamaModelDetails {
+// modelMetaFromConfig fetches the ModelConfig for `name` and derives both the
+// Ollama details block and capability list. Returns zero values when the model
+// is not configured.
+func modelMetaFromConfig(bcl *config.ModelConfigLoader, name string) (schema.OllamaModelDetails, []string) {
 	configName := strings.Split(name, ":")[0]
 	cfg, exists := bcl.GetModelConfig(configName)
 	if !exists {
-		return schema.OllamaModelDetails{}
+		return schema.OllamaModelDetails{}, nil
 	}
-	return modelDetailsFromModelConfig(&cfg)
+	return modelDetailsFromModelConfig(&cfg), modelCapabilities(&cfg)
 }
 
 func modelDetailsFromModelConfig(cfg *config.ModelConfig) schema.OllamaModelDetails {
-	return schema.OllamaModelDetails{
-		Format: "gguf",
-		Family: cfg.Backend,
+	family := cfg.Backend
+	details := schema.OllamaModelDetails{
+		Format:            "gguf",
+		Family:            family,
+		ParameterSize:     extractParameterSize(cfg.Model),
+		QuantizationLevel: extractQuantizationLevel(cfg.Model),
 	}
+	if family != "" {
+		details.Families = []string{family}
+	}
+	return details
+}
+
+// modelInfoFromModelConfig returns a small map of model_info entries derived
+// from the LocalAI ModelConfig. Ollama clients use this map for architecture
+// and context-length information; we expose what we can without loading the
+// model.
+func modelInfoFromModelConfig(cfg *config.ModelConfig) map[string]any {
+	info := map[string]any{}
+	if cfg.Backend != "" {
+		info["general.architecture"] = cfg.Backend
+	}
+	if cfg.ContextSize != nil && *cfg.ContextSize > 0 {
+		info["general.context_length"] = *cfg.ContextSize
+	}
+	if len(info) == 0 {
+		return nil
+	}
+	return info
 }