Skip to content

Commit bc3fb16

Browse files
localai-botmudler
localai-bot
and
mudler
authored
feat(ollama): report model capabilities + details on /api/tags and /api/show (#9766)
Ollama-compatible clients (Open WebUI, Enchanted, ollama-grid-search, etc.) rely on the `capabilities` list and `details.{parameter_size, quantization_level,families}` fields returned by /api/tags and /api/show to decide which models are eligible for a given task -- for example to filter the "embedding model" picker. Upstream Ollama returns these; LocalAI's compat layer was leaving them empty, so embedding models were silently rejected by clients that only allow chat models for chat and only allow embedding models for embeddings. This wires up the existing config signals already present in ModelConfig: - modelCapabilities() derives the Ollama capability strings from the config: "embedding" (FLAG_EMBEDDINGS), "completion" (FLAG_CHAT / FLAG_COMPLETION), "vision" (explicit KnownUsecases bit or MMProj / multimodal template / backend media marker), "tools" (auto-detected ToolFormatMarkers, JSON/Response regex, XML format, grammar triggers), "thinking" (ReasoningConfig with reasoning not disabled) and "insert" (presence of a completion template). - modelDetailsFromModelConfig() now fills families, parameter_size and quantization_level. The latter two are parsed from the GGUF filename via regex -- conservative tokens only (Q*/IQ*/F16/F32/BF16 and \d+(\.\d+)?[BM] surrounded by separators) so we don't accidentally match "Qwen3" as "3B". - modelInfoFromModelConfig() exposes general.architecture and general.context_length in the new ShowResponse.model_info map. Note: HasUsecases(FLAG_VISION) cannot be used directly -- GuessUsecases has no FLAG_VISION case and returns true at the end for any chat model. hasVisionSupport() instead reads KnownUsecases explicitly plus MMProj / template / media-marker signals. Tests are written first (TDD) using Ginkgo/Gomega -- DescribeTable for the capability mapping (embedding-only, chat, vision, thinking, tools via markers, tools via JSON regex, no-capability rerank) plus integration tests against ShowModelEndpoint that round-trip JSON through a real ModelConfigLoader populated from a temp YAML file. Fixes #9760. Assisted-by: Claude Code:claude-opus-4-7 Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 78722ca commit bc3fb16

5 files changed

Lines changed: 453 additions & 41 deletions

File tree

‎core/http/endpoints/ollama/capabilities.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package ollama
2+
3+
import (
4+
"regexp"
5+
"strings"
6+
7+
"github.com/mudler/LocalAI/core/config"
8+
)
9+
10+
// modelCapabilities maps a LocalAI ModelConfig to the Ollama capability strings
11+
// (https://github.com/ollama/ollama/blob/main/docs/api.md#show-model-information).
12+
//
13+
// Ollama clients use these to decide which models are eligible for a given task
14+
// (e.g. only allow embedding models in an "embedding model" picker). Returning
15+
// an empty list makes clients assume "completion" everywhere, which is wrong
16+
// for embedding/rerank/audio backends — see issue #9760.
17+
func modelCapabilities(cfg *config.ModelConfig) []string {
18+
if cfg == nil {
19+
return nil
20+
}
21+
22+
var caps []string
23+
24+
if cfg.HasUsecases(config.FLAG_EMBEDDINGS) {
25+
caps = append(caps, "embedding")
26+
}
27+
28+
chatCapable := cfg.HasUsecases(config.FLAG_CHAT) || cfg.HasUsecases(config.FLAG_COMPLETION)
29+
if chatCapable {
30+
caps = append(caps, "completion")
31+
}
32+
33+
if chatCapable && hasVisionSupport(cfg) {
34+
caps = append(caps, "vision")
35+
}
36+
37+
if chatCapable && hasToolSupport(cfg) {
38+
caps = append(caps, "tools")
39+
}
40+
41+
if chatCapable && hasThinkingSupport(cfg) {
42+
caps = append(caps, "thinking")
43+
}
44+
45+
if chatCapable && cfg.TemplateConfig.Completion != "" {
46+
caps = append(caps, "insert")
47+
}
48+
49+
return caps
50+
}
51+
52+
// hasVisionSupport reports whether the model can accept image inputs. We avoid
53+
// cfg.HasUsecases(FLAG_VISION) because GuessUsecases has no FLAG_VISION case
54+
// and returns true for any chat model — see core/config/model_config.go. Instead
55+
// we look for explicit signals: KnownUsecases bit, multimodal projector, or
56+
// template/backend-reported multimodal markers.
57+
func hasVisionSupport(cfg *config.ModelConfig) bool {
58+
if cfg.KnownUsecases != nil && (*cfg.KnownUsecases&config.FLAG_VISION) == config.FLAG_VISION {
59+
return true
60+
}
61+
if cfg.MMProj != "" {
62+
return true
63+
}
64+
if cfg.TemplateConfig.Multimodal != "" {
65+
return true
66+
}
67+
if cfg.MediaMarker != "" {
68+
return true
69+
}
70+
return false
71+
}
72+
73+
// hasToolSupport reports whether the model is wired up for tool / function calling.
74+
// We look for any of the explicit configuration knobs LocalAI uses to drive
75+
// function-call extraction (regex match, response regex, grammar triggers, XML
76+
// format) or for the auto-detected tool-format markers populated by the
77+
// llama.cpp backend during model load.
78+
func hasToolSupport(cfg *config.ModelConfig) bool {
79+
fc := cfg.FunctionsConfig
80+
if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
81+
return true
82+
}
83+
if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
84+
return true
85+
}
86+
if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
87+
return true
88+
}
89+
if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
90+
return true
91+
}
92+
return false
93+
}
94+
95+
// hasThinkingSupport reports whether the model has reasoning / thinking enabled.
96+
// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
97+
// when the backend probe reports that the model supports thinking.
98+
func hasThinkingSupport(cfg *config.ModelConfig) bool {
99+
rc := cfg.ReasoningConfig
100+
if rc.DisableReasoning != nil && !*rc.DisableReasoning {
101+
return true
102+
}
103+
if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
104+
// Explicit thinking markers imply support unless explicitly disabled.
105+
return rc.DisableReasoning == nil || !*rc.DisableReasoning
106+
}
107+
return false
108+
}
109+
110+
// quantRegex matches GGUF-style quantization suffixes (Q4_K_M, Q8_0, IQ3_XS, F16, ...).
111+
// Matches the convention used by GGUF tooling and what ggml-org/llama.cpp report.
112+
var quantRegex = regexp.MustCompile(`(?i)(IQ\d+(?:_[A-Z0-9]+)*|Q\d+(?:_[A-Z0-9]+)*|F16|F32|BF16)`)
113+
114+
// paramSizeRegex matches a parameter-size token surrounded by separators
115+
// (e.g. "-7B-", "_3b.", ".70B-"). Avoids matching the "7" inside "Qwen3".
116+
var paramSizeRegex = regexp.MustCompile(`(?i)(?:^|[-_.])(\d+(?:\.\d+)?[BM])(?:[-_.]|$)`)
117+
118+
// extractQuantizationLevel pulls the quantization tag from the model filename.
119+
// Returns the uppercased token (e.g. "Q4_K_M") or "" when not present.
120+
func extractQuantizationLevel(modelFile string) string {
121+
if modelFile == "" {
122+
return ""
123+
}
124+
base := strings.TrimSuffix(modelFile, ".gguf")
125+
if m := quantRegex.FindString(base); m != "" {
126+
return strings.ToUpper(m)
127+
}
128+
return ""
129+
}
130+
131+
// extractParameterSize pulls the parameter count from the model filename.
132+
// Returns "" when no recognizable token is present.
133+
func extractParameterSize(modelFile string) string {
134+
if modelFile == "" {
135+
return ""
136+
}
137+
base := strings.TrimSuffix(modelFile, ".gguf")
138+
if m := paramSizeRegex.FindStringSubmatch(base); len(m) > 1 {
139+
return strings.ToUpper(m[1])
140+
}
141+
return ""
142+
}

‎core/http/endpoints/ollama/capabilities_test.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package ollama
2+
3+
import (
4+
"github.com/mudler/LocalAI/core/config"
5+
"github.com/mudler/LocalAI/pkg/functions"
6+
"github.com/mudler/LocalAI/pkg/reasoning"
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
)
10+
11+
func boolPtr(b bool) *bool { return &b }
12+
13+
func withKnownUsecases(cfg config.ModelConfig, flags ...string) config.ModelConfig {
14+
cfg.KnownUsecaseStrings = flags
15+
cfg.KnownUsecases = config.GetUsecasesFromYAML(flags)
16+
return cfg
17+
}
18+
19+
var _ = Describe("modelCapabilities", func() {
20+
DescribeTable("derives Ollama capability strings from a ModelConfig",
21+
func(cfg config.ModelConfig, expected []string) {
22+
caps := modelCapabilities(&cfg)
23+
if len(expected) == 0 {
24+
Expect(caps).To(BeEmpty())
25+
return
26+
}
27+
Expect(caps).To(ConsistOf(expected))
28+
},
29+
Entry("an embedding-only model exposes the embedding capability",
30+
config.ModelConfig{
31+
Name: "embed-model",
32+
Backend: "llama-cpp",
33+
Embeddings: boolPtr(true),
34+
},
35+
[]string{"embedding"},
36+
),
37+
Entry("a chat-template model exposes the completion capability",
38+
config.ModelConfig{
39+
Name: "chat-model",
40+
Backend: "llama-cpp",
41+
TemplateConfig: config.TemplateConfig{
42+
Chat: "{{ .Input }}",
43+
},
44+
},
45+
[]string{"completion"},
46+
),
47+
Entry("a vision-capable chat model exposes completion + vision",
48+
withKnownUsecases(config.ModelConfig{
49+
Name: "vision-model",
50+
Backend: "llama-cpp",
51+
TemplateConfig: config.TemplateConfig{
52+
Chat: "{{ .Input }}",
53+
Multimodal: "<__media__>",
54+
},
55+
}, "FLAG_CHAT", "FLAG_VISION"),
56+
[]string{"completion", "vision"},
57+
),
58+
Entry("a model with reasoning enabled exposes the thinking capability",
59+
config.ModelConfig{
60+
Name: "thinking-model",
61+
Backend: "llama-cpp",
62+
TemplateConfig: config.TemplateConfig{
63+
Chat: "{{ .Input }}",
64+
},
65+
ReasoningConfig: reasoning.Config{
66+
DisableReasoning: boolPtr(false),
67+
},
68+
},
69+
[]string{"completion", "thinking"},
70+
),
71+
Entry("a model with detected tool-format markers exposes the tools capability",
72+
config.ModelConfig{
73+
Name: "tools-model",
74+
Backend: "llama-cpp",
75+
TemplateConfig: config.TemplateConfig{
76+
Chat: "{{ .Input }}",
77+
},
78+
FunctionsConfig: functions.FunctionsConfig{
79+
ToolFormatMarkers: &functions.ToolFormatMarkers{FormatType: "json_native"},
80+
},
81+
},
82+
[]string{"completion", "tools"},
83+
),
84+
Entry("a model with an explicit JSON regex match exposes the tools capability",
85+
config.ModelConfig{
86+
Name: "tools-regex-model",
87+
Backend: "llama-cpp",
88+
TemplateConfig: config.TemplateConfig{
89+
Chat: "{{ .Input }}",
90+
},
91+
FunctionsConfig: functions.FunctionsConfig{
92+
JSONRegexMatch: []string{`(?s).*`},
93+
},
94+
},
95+
[]string{"completion", "tools"},
96+
),
97+
Entry("a pure backend-only model (no template, no embeddings) reports no capabilities",
98+
config.ModelConfig{
99+
Name: "rerank-model",
100+
Backend: "rerankers",
101+
},
102+
[]string{},
103+
),
104+
)
105+
})
106+
107+
var _ = Describe("modelDetailsFromModelConfig", func() {
108+
It("reports gguf format and llama-cpp family/families for a llama-cpp model", func() {
109+
cfg := config.ModelConfig{
110+
Name: "llama",
111+
Backend: "llama-cpp",
112+
}
113+
details := modelDetailsFromModelConfig(&cfg)
114+
Expect(details.Format).To(Equal("gguf"))
115+
Expect(details.Family).To(Equal("llama-cpp"))
116+
Expect(details.Families).To(ConsistOf("llama-cpp"))
117+
})
118+
119+
It("extracts quantization_level from the model filename when present", func() {
120+
cfg := config.ModelConfig{
121+
Name: "qwen-q4",
122+
Backend: "llama-cpp",
123+
}
124+
cfg.Model = "Qwen3-4B-Instruct-Q4_K_M.gguf"
125+
details := modelDetailsFromModelConfig(&cfg)
126+
Expect(details.QuantizationLevel).To(Equal("Q4_K_M"))
127+
})
128+
129+
It("extracts parameter_size from the model filename when present", func() {
130+
cfg := config.ModelConfig{
131+
Name: "qwen-4b",
132+
Backend: "llama-cpp",
133+
}
134+
cfg.Model = "Qwen3-4B-Instruct-Q4_K_M.gguf"
135+
details := modelDetailsFromModelConfig(&cfg)
136+
Expect(details.ParameterSize).To(Equal("4B"))
137+
})
138+
})

‎core/http/endpoints/ollama/models.go‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader) ec
3232

3333
digest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name)))
3434

35+
details, caps := modelMetaFromConfig(bcl, name)
3536
entry := schema.OllamaModelEntry{
36-
Name: ollamaName,
37-
Model: ollamaName,
38-
ModifiedAt: time.Now().UTC(),
39-
Size: 0,
40-
Digest: digest,
41-
Details: modelDetailsFromConfig(bcl, name),
37+
Name: ollamaName,
38+
Model: ollamaName,
39+
ModifiedAt: time.Now().UTC(),
40+
Size: 0,
41+
Digest: digest,
42+
Details: details,
43+
Capabilities: caps,
4244
}
4345
models = append(models, entry)
4446
}
@@ -72,10 +74,12 @@ func ShowModelEndpoint(bcl *config.ModelConfigLoader) echo.HandlerFunc {
7274
}
7375

7476
resp := schema.OllamaShowResponse{
75-
Modelfile: fmt.Sprintf("FROM %s", cfg.Model),
76-
Parameters: "",
77-
Template: cfg.TemplateConfig.Chat,
78-
Details: modelDetailsFromModelConfig(&cfg),
77+
Modelfile: fmt.Sprintf("FROM %s", cfg.Model),
78+
Parameters: "",
79+
Template: cfg.TemplateConfig.Chat,
80+
Details: modelDetailsFromModelConfig(&cfg),
81+
ModelInfo: modelInfoFromModelConfig(&cfg),
82+
Capabilities: modelCapabilities(&cfg),
7983
}
8084

8185
return c.JSON(200, resp)
@@ -95,14 +99,16 @@ func ListRunningEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader) e
9599
ollamaName += ":latest"
96100
}
97101

102+
details, caps := modelMetaFromConfig(bcl, name)
98103
entry := schema.OllamaPsEntry{
99-
Name: ollamaName,
100-
Model: ollamaName,
101-
Size: 0,
102-
Digest: fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name))),
103-
Details: modelDetailsFromConfig(bcl, name),
104-
ExpiresAt: time.Now().Add(24 * time.Hour).UTC(),
105-
SizeVRAM: 0,
104+
Name: ollamaName,
105+
Model: ollamaName,
106+
Size: 0,
107+
Digest: fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(name))),
108+
Details: details,
109+
ExpiresAt: time.Now().Add(24 * time.Hour).UTC(),
110+
SizeVRAM: 0,
111+
Capabilities: caps,
106112
}
107113
models = append(models, entry)
108114
}
@@ -125,18 +131,46 @@ func HeartbeatEndpoint() echo.HandlerFunc {
125131
}
126132
}
127133

128-
func modelDetailsFromConfig(bcl *config.ModelConfigLoader, name string) schema.OllamaModelDetails {
134+
// modelMetaFromConfig fetches the ModelConfig for `name` and derives both the
135+
// Ollama details block and capability list. Returns zero values when the model
136+
// is not configured.
137+
func modelMetaFromConfig(bcl *config.ModelConfigLoader, name string) (schema.OllamaModelDetails, []string) {
129138
configName := strings.Split(name, ":")[0]
130139
cfg, exists := bcl.GetModelConfig(configName)
131140
if !exists {
132-
return schema.OllamaModelDetails{}
141+
return schema.OllamaModelDetails{}, nil
133142
}
134-
return modelDetailsFromModelConfig(&cfg)
143+
return modelDetailsFromModelConfig(&cfg), modelCapabilities(&cfg)
135144
}
136145

137146
func modelDetailsFromModelConfig(cfg *config.ModelConfig) schema.OllamaModelDetails {
138-
return schema.OllamaModelDetails{
139-
Format: "gguf",
140-
Family: cfg.Backend,
147+
family := cfg.Backend
148+
details := schema.OllamaModelDetails{
149+
Format: "gguf",
150+
Family: family,
151+
ParameterSize: extractParameterSize(cfg.Model),
152+
QuantizationLevel: extractQuantizationLevel(cfg.Model),
141153
}
154+
if family != "" {
155+
details.Families = []string{family}
156+
}
157+
return details
158+
}
159+
160+
// modelInfoFromModelConfig returns a small map of model_info entries derived
161+
// from the LocalAI ModelConfig. Ollama clients use this map for architecture
162+
// and context-length information; we expose what we can without loading the
163+
// model.
164+
func modelInfoFromModelConfig(cfg *config.ModelConfig) map[string]any {
165+
info := map[string]any{}
166+
if cfg.Backend != "" {
167+
info["general.architecture"] = cfg.Backend
168+
}
169+
if cfg.ContextSize != nil && *cfg.ContextSize > 0 {
170+
info["general.context_length"] = *cfg.ContextSize
171+
}
172+
if len(info) == 0 {
173+
return nil
174+
}
175+
return info
142176
}

0 commit comments

Comments
 (0)