Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions core/gallery/importers/mlx.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package importers
import (
"encoding/json"
"path/filepath"
"slices"
"strings"

"github.com/mudler/LocalAI/core/config"
Expand Down Expand Up @@ -31,7 +32,7 @@ func (i *MLXImporter) Match(details Details) bool {
}

b, ok := preferencesMap["backend"].(string)
if ok && b == "mlx" || b == "mlx-vlm" {
if ok && slices.Contains([]string{"mlx", "mlx-vlm", "mlx-audio"}, b) {
return true
}

Expand Down Expand Up @@ -71,27 +72,40 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) {
// (issue #10269). Send them to the mlx-vlm backend, which applies the
// processor-aware chat template.
backend := "mlx"
if details.HuggingFace != nil && details.HuggingFace.PipelineTag == "image-text-to-text" {
backend = "mlx-vlm"
usecases := []string{config.UsecaseChat}
useTokenizerTemplate := true
if details.HuggingFace != nil {
switch details.HuggingFace.PipelineTag {
case "image-text-to-text":
backend = "mlx-vlm"
case "text-to-speech":
backend = "mlx-audio"
usecases = []string{config.UsecaseTTS}
useTokenizerTemplate = false
}
}
// An explicit backend preference always wins.
b, ok := preferencesMap["backend"].(string)
if ok {
backend = b
if backend == "mlx-audio" {
usecases = []string{config.UsecaseTTS}
useTokenizerTemplate = false
}
}

modelConfig := config.ModelConfig{
Name: name,
Description: description,
KnownUsecaseStrings: []string{config.UsecaseChat},
KnownUsecaseStrings: usecases,
Backend: backend,
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: LocalModelPath(details.URI),
},
},
TemplateConfig: config.TemplateConfig{
UseTokenizerTemplate: true,
UseTokenizerTemplate: useTokenizerTemplate,
},
}

Expand Down
42 changes: 42 additions & 0 deletions core/gallery/importers/mlx_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,16 @@ var _ = Describe("MLXImporter", func() {
Expect(result).To(BeTrue())
})

It("should match when backend preference is mlx-audio", func() {
preferences := json.RawMessage(`{"backend": "mlx-audio"}`)
details := importers.Details{
URI: "https://example.com/model",
Preferences: preferences,
}

Expect(importer.Match(details)).To(BeTrue())
})

It("should not match when URI does not contain mlx-community/ and no backend preference", func() {
details := importers.Details{
URI: "https://huggingface.co/other-org/test-model",
Expand Down Expand Up @@ -123,6 +133,21 @@ var _ = Describe("MLXImporter", func() {
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm"))
})

It("should configure explicit mlx-audio imports for text-to-speech", func() {
preferences := json.RawMessage(`{"backend": "mlx-audio"}`)
details := importers.Details{
URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit",
Preferences: preferences,
}

modelConfig, err := importer.Import(details)

Expect(err).ToNot(HaveOccurred())
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio"))
Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts"))
Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true"))
})

It("should auto-route vision-language models to the mlx-vlm backend", func() {
// gemma-4 E4B and similar VLMs declare pipeline_tag
// "image-text-to-text" on HuggingFace. The text-only mlx-lm
Expand All @@ -143,6 +168,23 @@ var _ = Describe("MLXImporter", func() {
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm"))
})

It("should auto-route text-to-speech models to the mlx-audio backend", func() {
details := importers.Details{
URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit",
HuggingFace: &hfapi.ModelDetails{
ModelID: "mlx-community/Kokoro-82M-4bit",
PipelineTag: "text-to-speech",
},
}

modelConfig, err := importer.Import(details)

Expect(err).ToNot(HaveOccurred())
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio"))
Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts"))
Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true"))
})

It("should keep text-only models on the plain mlx backend", func() {
details := importers.Details{
URI: "https://huggingface.co/mlx-community/Llama-3.2-1B-Instruct-4bit",
Expand Down
1 change: 1 addition & 0 deletions core/http/endpoints/localai/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{
{Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"},
{Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"},
// TTS
{Name: "mlx-audio", Modality: "tts", AutoDetect: false, Description: "MLX-Audio text-to-speech models (auto-detected; pref-only fallback)"},
{Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"},
{Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"},
{Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"},
Expand Down
1 change: 1 addition & 0 deletions core/http/endpoints/localai/backend_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ var _ = Describe("Backend Endpoints", func() {
expectPrefOnly("tinygrad", "text")
expectPrefOnly("trl", "text")
expectPrefOnly("mlx-vlm", "text")
expectPrefOnly("mlx-audio", "tts")
expectPrefOnly("whisperx", "asr")
expectPrefOnly("crispasr", "asr")
expectPrefOnly("kokoros", "tts")
Expand Down
6 changes: 6 additions & 0 deletions docs/content/getting-started/models.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ The WebUI provides a powerful model import interface that supports both simple a
- Custom preferences
5. Click "Import Model" to start the import process

Repositories under `mlx-community` are imported with the native MLX backend.
LocalAI uses Hugging Face's pipeline metadata to select `mlx-vlm` for
vision-language models and `mlx-audio` for text-to-speech models; other MLX
repositories use `mlx`. An explicit backend selection in the import form always
overrides this automatic routing.

### Advanced Import Mode

For full control over model configuration:
Expand Down
Loading