fix(gallery): write installed inference defaults under parameters so the loader reads them (#11230)

core/gallery/model_artifacts_inference_defaults_test.go

@@ -0,0 +1,63 @@
+package gallery_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	. "github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+var _ = Describe("gallery inference-default persistence", func() {
+	It("persists inference defaults under parameters so the loader reads them back", func() {
+		modelsPath, err := os.MkdirTemp("", "inference-defaults")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(modelsPath)
+
+		systemState, err := system.GetSystemState(system.WithModelPath(modelsPath))
+		Expect(err).ToNot(HaveOccurred())
+
+		// A qwen3.5 name makes ApplyInferenceDefaults fill in the recommended
+		// sampling parameters (repeat_penalty=1, presence_penalty=1.5, min_p=0).
+		// Those belong under the parameters: key — ModelConfig embeds
+		// schema.PredictionOptions with `yaml:"parameters"`, so the loader only
+		// reads them from that submap (#11230). The install is fully offline:
+		// the definition declares no files, so InstallModel just writes the YAML.
+		definition := &ModelConfig{ConfigFile: `backend: transformers
+parameters:
+  model: owner/repo
+`}
+
+		_, err = InstallModel(context.TODO(), systemState, "qwen3.5-managed", definition, map[string]any{}, func(string, string, string, float64) {}, false)
+		Expect(err).ToNot(HaveOccurred())
+
+		data, err := os.ReadFile(filepath.Join(modelsPath, "qwen3.5-managed.yaml"))
+		Expect(err).ToNot(HaveOccurred())
+
+		// The defaults must survive a round-trip through the typed loader.
+		var reloaded config.ModelConfig
+		Expect(yaml.Unmarshal(data, &reloaded)).To(Succeed())
+		Expect(reloaded.PresencePenalty).To(BeNumerically("==", 1.5))
+		Expect(reloaded.RepeatPenalty).To(BeNumerically("==", 1))
+		Expect(reloaded.MinP).NotTo(BeNil())
+		Expect(reloaded.Temperature).NotTo(BeNil())
+
+		// They must live under parameters:, never at the top level, or they are
+		// silently dropped on reload.
+		var raw map[string]any
+		Expect(yaml.Unmarshal(data, &raw)).To(Succeed())
+		Expect(raw).NotTo(HaveKey("presence_penalty"))
+		Expect(raw).NotTo(HaveKey("repeat_penalty"))
+		Expect(raw).NotTo(HaveKey("min_p"))
+		parameters, ok := raw["parameters"].(map[string]any)
+		Expect(ok).To(BeTrue())
+		Expect(parameters).To(HaveKey("presence_penalty"))
+		Expect(parameters).To(HaveKey("repeat_penalty"))
+		Expect(parameters).To(HaveKey("min_p"))
+	})
+})

core/gallery/models.go

@@ -270,36 +270,47 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
 		lconfig.ApplyInferenceDefaults(&modelConfig, name, modelConfig.Model)
 
 		// Merge inference defaults into configMap so they are persisted without losing unknown fields.
+		// These sampling parameters live under the `parameters:` key on disk: ModelConfig
+		// embeds schema.PredictionOptions with `yaml:"parameters"`, so the loader only reads
+		// them from that submap. Writing them at the top level produced keys the loader never
+		// read back, leaving the persisted defaults inert on reload (#11230).
+		params, ok := configMap["parameters"].(map[string]any)
+		if !ok {
+			params = make(map[string]any)
+		}
 		if modelConfig.Temperature != nil {
-			if _, exists := configMap["temperature"]; !exists {
-				configMap["temperature"] = *modelConfig.Temperature
+			if _, exists := params["temperature"]; !exists {
+				params["temperature"] = *modelConfig.Temperature
 			}
 		}
 		if modelConfig.TopP != nil {
-			if _, exists := configMap["top_p"]; !exists {
-				configMap["top_p"] = *modelConfig.TopP
+			if _, exists := params["top_p"]; !exists {
+				params["top_p"] = *modelConfig.TopP
 			}
 		}
 		if modelConfig.TopK != nil {
-			if _, exists := configMap["top_k"]; !exists {
-				configMap["top_k"] = *modelConfig.TopK
+			if _, exists := params["top_k"]; !exists {
+				params["top_k"] = *modelConfig.TopK
 			}
 		}
 		if modelConfig.MinP != nil {
-			if _, exists := configMap["min_p"]; !exists {
-				configMap["min_p"] = *modelConfig.MinP
+			if _, exists := params["min_p"]; !exists {
+				params["min_p"] = *modelConfig.MinP
 			}
 		}
 		if modelConfig.RepeatPenalty != 0 {
-			if _, exists := configMap["repeat_penalty"]; !exists {
-				configMap["repeat_penalty"] = modelConfig.RepeatPenalty
+			if _, exists := params["repeat_penalty"]; !exists {
+				params["repeat_penalty"] = modelConfig.RepeatPenalty
 			}
 		}
 		if modelConfig.PresencePenalty != 0 {
-			if _, exists := configMap["presence_penalty"]; !exists {
-				configMap["presence_penalty"] = modelConfig.PresencePenalty
+			if _, exists := params["presence_penalty"]; !exists {
+				params["presence_penalty"] = modelConfig.PresencePenalty
 			}
 		}
+		if len(params) > 0 {
+			configMap["parameters"] = params
+		}
 
 		// Re-marshal from configMap to preserve unknown fields
 		updatedConfigYAML, err = yaml.Marshal(configMap)
@@ -494,4 +505,4 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
 		}
 	}
 	return nil
-}
+}