Skip to content

Commit ae4b758

Browse files
mudlermudler
committed
feat: add fine-tuning endpoint
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 9cdbd89 commit ae4b758

8 files changed

Lines changed: 2801 additions & 0 deletions

File tree

‎core/gallery/importers/local.go‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
package importers
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
10+
"github.com/mudler/LocalAI/core/config"
11+
"github.com/mudler/xlog"
12+
)
13+
14+
// ImportLocalPath scans a local directory for exported model files and produces
15+
// a config.ModelConfig with the correct backend, model path, and options.
16+
// Paths in the returned config are relative to modelsPath when possible so that
17+
// the YAML config remains portable.
18+
//
19+
// Detection order:
20+
// 1. GGUF files (*.gguf) — uses llama-cpp backend
21+
// 2. LoRA adapter (adapter_config.json) — uses transformers backend with lora_adapter
22+
// 3. Merged model (*.safetensors or pytorch_model*.bin + config.json) — uses transformers backend
23+
func ImportLocalPath(dirPath, name string) (*config.ModelConfig, error) {
24+
// Make paths relative to the models directory (parent of dirPath)
25+
// so config YAML stays portable.
26+
modelsDir := filepath.Dir(dirPath)
27+
relPath := func(absPath string) string {
28+
if rel, err := filepath.Rel(modelsDir, absPath); err == nil {
29+
return rel
30+
}
31+
return absPath
32+
}
33+
34+
// 1. GGUF: check dirPath and dirPath_gguf/ (Unsloth convention)
35+
ggufFile := findGGUF(dirPath)
36+
if ggufFile == "" {
37+
ggufSubdir := dirPath + "_gguf"
38+
ggufFile = findGGUF(ggufSubdir)
39+
}
40+
if ggufFile != "" {
41+
xlog.Info("ImportLocalPath: detected GGUF model", "path", ggufFile)
42+
cfg := &config.ModelConfig{
43+
Name: name,
44+
Backend: "llama-cpp",
45+
KnownUsecaseStrings: []string{"chat"},
46+
Options: []string{"use_jinja:true"},
47+
}
48+
cfg.Model = relPath(ggufFile)
49+
cfg.TemplateConfig.UseTokenizerTemplate = true
50+
cfg.Description = buildDescription(dirPath, "GGUF")
51+
return cfg, nil
52+
}
53+
54+
// 2. LoRA adapter: look for adapter_config.json
55+
56+
adapterConfigPath := filepath.Join(dirPath, "adapter_config.json")
57+
if fileExists(adapterConfigPath) {
58+
xlog.Info("ImportLocalPath: detected LoRA adapter", "path", dirPath)
59+
baseModel := readBaseModel(dirPath)
60+
cfg := &config.ModelConfig{
61+
Name: name,
62+
Backend: "transformers",
63+
KnownUsecaseStrings: []string{"chat"},
64+
}
65+
cfg.Model = baseModel
66+
cfg.TemplateConfig.UseTokenizerTemplate = true
67+
cfg.LLMConfig.LoraAdapter = relPath(dirPath)
68+
cfg.Description = buildDescription(dirPath, "LoRA adapter")
69+
return cfg, nil
70+
}
71+
72+
// Also check for adapter_model.safetensors or adapter_model.bin without adapter_config.json
73+
if fileExists(filepath.Join(dirPath, "adapter_model.safetensors")) || fileExists(filepath.Join(dirPath, "adapter_model.bin")) {
74+
xlog.Info("ImportLocalPath: detected LoRA adapter (by model files)", "path", dirPath)
75+
baseModel := readBaseModel(dirPath)
76+
cfg := &config.ModelConfig{
77+
Name: name,
78+
Backend: "transformers",
79+
KnownUsecaseStrings: []string{"chat"},
80+
}
81+
cfg.Model = baseModel
82+
cfg.TemplateConfig.UseTokenizerTemplate = true
83+
cfg.LLMConfig.LoraAdapter = relPath(dirPath)
84+
cfg.Description = buildDescription(dirPath, "LoRA adapter")
85+
return cfg, nil
86+
}
87+
88+
// 3. Merged model: *.safetensors or pytorch_model*.bin + config.json
89+
if fileExists(filepath.Join(dirPath, "config.json")) && (hasFileWithSuffix(dirPath, ".safetensors") || hasFileWithPrefix(dirPath, "pytorch_model")) {
90+
xlog.Info("ImportLocalPath: detected merged model", "path", dirPath)
91+
cfg := &config.ModelConfig{
92+
Name: name,
93+
Backend: "transformers",
94+
KnownUsecaseStrings: []string{"chat"},
95+
}
96+
cfg.Model = relPath(dirPath)
97+
cfg.TemplateConfig.UseTokenizerTemplate = true
98+
cfg.Description = buildDescription(dirPath, "merged model")
99+
return cfg, nil
100+
}
101+
102+
return nil, fmt.Errorf("could not detect model format in directory %s", dirPath)
103+
}
104+
105+
// findGGUF returns the path to the first .gguf file found in dir, or "".
106+
func findGGUF(dir string) string {
107+
entries, err := os.ReadDir(dir)
108+
if err != nil {
109+
return ""
110+
}
111+
for _, e := range entries {
112+
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") {
113+
return filepath.Join(dir, e.Name())
114+
}
115+
}
116+
return ""
117+
}
118+
119+
// readBaseModel reads the base model name from adapter_config.json or export_metadata.json.
120+
func readBaseModel(dirPath string) string {
121+
// Try adapter_config.json → base_model_name_or_path (TRL writes this)
122+
if data, err := os.ReadFile(filepath.Join(dirPath, "adapter_config.json")); err == nil {
123+
var ac map[string]any
124+
if json.Unmarshal(data, &ac) == nil {
125+
if bm, ok := ac["base_model_name_or_path"].(string); ok && bm != "" {
126+
return bm
127+
}
128+
}
129+
}
130+
131+
// Try export_metadata.json → base_model (Unsloth writes this)
132+
if data, err := os.ReadFile(filepath.Join(dirPath, "export_metadata.json")); err == nil {
133+
var meta map[string]any
134+
if json.Unmarshal(data, &meta) == nil {
135+
if bm, ok := meta["base_model"].(string); ok && bm != "" {
136+
return bm
137+
}
138+
}
139+
}
140+
141+
return ""
142+
}
143+
144+
// buildDescription creates a human-readable description using available metadata.
145+
func buildDescription(dirPath, formatLabel string) string {
146+
base := ""
147+
148+
// Try adapter_config.json
149+
if data, err := os.ReadFile(filepath.Join(dirPath, "adapter_config.json")); err == nil {
150+
var ac map[string]any
151+
if json.Unmarshal(data, &ac) == nil {
152+
if bm, ok := ac["base_model_name_or_path"].(string); ok && bm != "" {
153+
base = bm
154+
}
155+
}
156+
}
157+
158+
// Try export_metadata.json
159+
if base == "" {
160+
if data, err := os.ReadFile(filepath.Join(dirPath, "export_metadata.json")); err == nil {
161+
var meta map[string]any
162+
if json.Unmarshal(data, &meta) == nil {
163+
if bm, ok := meta["base_model"].(string); ok && bm != "" {
164+
base = bm
165+
}
166+
}
167+
}
168+
}
169+
170+
if base != "" {
171+
return fmt.Sprintf("Fine-tuned from %s (%s)", base, formatLabel)
172+
}
173+
return fmt.Sprintf("Fine-tuned model (%s)", formatLabel)
174+
}
175+
176+
func fileExists(path string) bool {
177+
info, err := os.Stat(path)
178+
return err == nil && !info.IsDir()
179+
}
180+
181+
func hasFileWithSuffix(dir, suffix string) bool {
182+
entries, err := os.ReadDir(dir)
183+
if err != nil {
184+
return false
185+
}
186+
for _, e := range entries {
187+
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), suffix) {
188+
return true
189+
}
190+
}
191+
return false
192+
}
193+
194+
func hasFileWithPrefix(dir, prefix string) bool {
195+
entries, err := os.ReadDir(dir)
196+
if err != nil {
197+
return false
198+
}
199+
for _, e := range entries {
200+
if !e.IsDir() && strings.HasPrefix(e.Name(), prefix) {
201+
return true
202+
}
203+
}
204+
return false
205+
}

‎core/gallery/importers/local_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package importers_test
2+
3+
import (
4+
"encoding/json"
5+
"os"
6+
"path/filepath"
7+
8+
. "github.com/onsi/ginkgo/v2"
9+
. "github.com/onsi/gomega"
10+
11+
"github.com/mudler/LocalAI/core/gallery/importers"
12+
)
13+
14+
var _ = Describe("ImportLocalPath", func() {
15+
var tmpDir string
16+
17+
BeforeEach(func() {
18+
var err error
19+
tmpDir, err = os.MkdirTemp("", "importers-local-test")
20+
Expect(err).ToNot(HaveOccurred())
21+
})
22+
23+
AfterEach(func() {
24+
os.RemoveAll(tmpDir)
25+
})
26+
27+
Context("GGUF detection", func() {
28+
It("detects a GGUF file in the directory", func() {
29+
modelDir := filepath.Join(tmpDir, "my-model")
30+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
31+
Expect(os.WriteFile(filepath.Join(modelDir, "model-q4_k_m.gguf"), []byte("fake"), 0644)).To(Succeed())
32+
33+
cfg, err := importers.ImportLocalPath(modelDir, "my-model")
34+
Expect(err).ToNot(HaveOccurred())
35+
Expect(cfg.Backend).To(Equal("llama-cpp"))
36+
Expect(cfg.Model).To(ContainSubstring(".gguf"))
37+
Expect(cfg.TemplateConfig.UseTokenizerTemplate).To(BeTrue())
38+
Expect(cfg.KnownUsecaseStrings).To(ContainElement("chat"))
39+
Expect(cfg.Options).To(ContainElement("use_jinja:true"))
40+
})
41+
42+
It("detects GGUF in _gguf subdirectory", func() {
43+
modelDir := filepath.Join(tmpDir, "my-model")
44+
ggufDir := modelDir + "_gguf"
45+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
46+
Expect(os.MkdirAll(ggufDir, 0755)).To(Succeed())
47+
Expect(os.WriteFile(filepath.Join(ggufDir, "model.gguf"), []byte("fake"), 0644)).To(Succeed())
48+
49+
cfg, err := importers.ImportLocalPath(modelDir, "my-model")
50+
Expect(err).ToNot(HaveOccurred())
51+
Expect(cfg.Backend).To(Equal("llama-cpp"))
52+
})
53+
})
54+
55+
Context("LoRA adapter detection", func() {
56+
It("detects LoRA adapter via adapter_config.json", func() {
57+
modelDir := filepath.Join(tmpDir, "lora-model")
58+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
59+
60+
adapterConfig := map[string]any{
61+
"base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
62+
"peft_type": "LORA",
63+
}
64+
data, _ := json.Marshal(adapterConfig)
65+
Expect(os.WriteFile(filepath.Join(modelDir, "adapter_config.json"), data, 0644)).To(Succeed())
66+
67+
cfg, err := importers.ImportLocalPath(modelDir, "lora-model")
68+
Expect(err).ToNot(HaveOccurred())
69+
Expect(cfg.Backend).To(Equal("transformers"))
70+
Expect(cfg.Model).To(Equal("meta-llama/Llama-2-7b-hf"))
71+
Expect(cfg.LLMConfig.LoraAdapter).To(Equal(modelDir))
72+
Expect(cfg.TemplateConfig.UseTokenizerTemplate).To(BeTrue())
73+
})
74+
75+
It("reads base model from export_metadata.json as fallback", func() {
76+
modelDir := filepath.Join(tmpDir, "lora-unsloth")
77+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
78+
79+
adapterConfig := map[string]any{"peft_type": "LORA"}
80+
data, _ := json.Marshal(adapterConfig)
81+
Expect(os.WriteFile(filepath.Join(modelDir, "adapter_config.json"), data, 0644)).To(Succeed())
82+
83+
metadata := map[string]any{"base_model": "unsloth/tinyllama-bnb-4bit"}
84+
data, _ = json.Marshal(metadata)
85+
Expect(os.WriteFile(filepath.Join(modelDir, "export_metadata.json"), data, 0644)).To(Succeed())
86+
87+
cfg, err := importers.ImportLocalPath(modelDir, "lora-unsloth")
88+
Expect(err).ToNot(HaveOccurred())
89+
Expect(cfg.Model).To(Equal("unsloth/tinyllama-bnb-4bit"))
90+
})
91+
})
92+
93+
Context("Merged model detection", func() {
94+
It("detects merged model with safetensors + config.json", func() {
95+
modelDir := filepath.Join(tmpDir, "merged")
96+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
97+
Expect(os.WriteFile(filepath.Join(modelDir, "config.json"), []byte("{}"), 0644)).To(Succeed())
98+
Expect(os.WriteFile(filepath.Join(modelDir, "model.safetensors"), []byte("fake"), 0644)).To(Succeed())
99+
100+
cfg, err := importers.ImportLocalPath(modelDir, "merged")
101+
Expect(err).ToNot(HaveOccurred())
102+
Expect(cfg.Backend).To(Equal("transformers"))
103+
Expect(cfg.Model).To(Equal(modelDir))
104+
Expect(cfg.TemplateConfig.UseTokenizerTemplate).To(BeTrue())
105+
})
106+
107+
It("detects merged model with pytorch_model files", func() {
108+
modelDir := filepath.Join(tmpDir, "merged-pt")
109+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
110+
Expect(os.WriteFile(filepath.Join(modelDir, "config.json"), []byte("{}"), 0644)).To(Succeed())
111+
Expect(os.WriteFile(filepath.Join(modelDir, "pytorch_model-00001-of-00002.bin"), []byte("fake"), 0644)).To(Succeed())
112+
113+
cfg, err := importers.ImportLocalPath(modelDir, "merged-pt")
114+
Expect(err).ToNot(HaveOccurred())
115+
Expect(cfg.Backend).To(Equal("transformers"))
116+
Expect(cfg.Model).To(Equal(modelDir))
117+
})
118+
})
119+
120+
Context("fallback", func() {
121+
It("returns error for empty directory", func() {
122+
modelDir := filepath.Join(tmpDir, "empty")
123+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
124+
125+
_, err := importers.ImportLocalPath(modelDir, "empty")
126+
Expect(err).To(HaveOccurred())
127+
Expect(err.Error()).To(ContainSubstring("could not detect model format"))
128+
})
129+
})
130+
131+
Context("description", func() {
132+
It("includes base model name in description", func() {
133+
modelDir := filepath.Join(tmpDir, "desc-test")
134+
Expect(os.MkdirAll(modelDir, 0755)).To(Succeed())
135+
136+
adapterConfig := map[string]any{
137+
"base_model_name_or_path": "TinyLlama/TinyLlama-1.1B",
138+
}
139+
data, _ := json.Marshal(adapterConfig)
140+
Expect(os.WriteFile(filepath.Join(modelDir, "adapter_config.json"), data, 0644)).To(Succeed())
141+
142+
cfg, err := importers.ImportLocalPath(modelDir, "desc-test")
143+
Expect(err).ToNot(HaveOccurred())
144+
Expect(cfg.Description).To(ContainSubstring("TinyLlama/TinyLlama-1.1B"))
145+
Expect(cfg.Description).To(ContainSubstring("Fine-tuned from"))
146+
})
147+
})
148+
})

0 commit comments

Comments
 (0)