fix(gallery): write installed inference defaults under parameters so the loader reads them back (#11230)

ModelConfig embeds schema.PredictionOptions with `yaml:"parameters"`, so
the loader only reads temperature/top_p/top_k/min_p/repeat_penalty/
presence_penalty from the parameters: submap. The gallery installer wrote
those family defaults at the top level of the model YAML, where nothing
reads them back, leaving every persisted default inert on reload.

Merge them into the parameters: submap instead (preserving any values the
config already sets there), and add a regression test that installs a
qwen3.5 model and asserts the defaults round-trip through the typed
loader rather than landing as inert top-level keys.

Signed-off-by: Anai-Guo <antai12232931@outlook.com>

Author: Anai-Guo

core/gallery/model_artifacts_inference_defaults_test.go

@@ -0,0 +1,80 @@
+package gallery_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/modelartifacts"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+var _ = Describe("gallery inference-default persistence", func() {
+	It("persists inference defaults under parameters so the loader reads them back", func() {
+		modelsPath := GinkgoT().TempDir()
+		state, err := system.GetSystemState(system.WithModelPath(modelsPath))
+		Expect(err).NotTo(HaveOccurred())
+		resolved := modelartifacts.Spec{
+			Name: "model", Target: "model",
+			Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
+			Resolved: &modelartifacts.Resolved{
+				Endpoint: "https://huggingface.co",
+				Revision: "0123456789abcdef0123456789abcdef01234567",
+				CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			},
+		}
+		fake := &fakeArtifactMaterializer{result: modelartifacts.Result{
+			Spec:         resolved,
+			RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
+		}}
+		definition := &gallery.ModelConfig{Name: "qwen3.5-managed", ConfigFile: `
+backend: transformers
+artifacts:
+  - name: model
+    target: model
+    source:
+      type: huggingface
+      repo: owner/repo
+parameters:
+  model: owner/repo
+`}
+
+		// A qwen3.5 name makes ApplyInferenceDefaults fill in the recommended
+		// sampling parameters (repeat_penalty=1, presence_penalty=1.5, min_p=0).
+		// Those belong under the parameters: key — ModelConfig embeds
+		// schema.PredictionOptions with `yaml:"parameters"`, so the loader only
+		// reads them from that submap (#11230).
+		_, err = gallery.InstallModel(context.Background(), state, "qwen3.5-managed", definition, nil, nil, false,
+			gallery.WithArtifactMaterializer(fake))
+		Expect(err).NotTo(HaveOccurred())
+
+		data, err := os.ReadFile(filepath.Join(modelsPath, "qwen3.5-managed.yaml"))
+		Expect(err).NotTo(HaveOccurred())
+
+		// The defaults must survive a round-trip through the typed loader.
+		var reloaded config.ModelConfig
+		Expect(yaml.Unmarshal(data, &reloaded)).To(Succeed())
+		Expect(reloaded.PresencePenalty).To(BeNumerically("==", 1.5))
+		Expect(reloaded.RepeatPenalty).To(BeNumerically("==", 1))
+		Expect(reloaded.MinP).NotTo(BeNil())
+		Expect(reloaded.Temperature).NotTo(BeNil())
+
+		// They must live under parameters:, never at the top level, or they are
+		// silently dropped on reload.
+		var raw map[string]any
+		Expect(yaml.Unmarshal(data, &raw)).To(Succeed())
+		Expect(raw).NotTo(HaveKey("presence_penalty"))
+		Expect(raw).NotTo(HaveKey("repeat_penalty"))
+		Expect(raw).NotTo(HaveKey("min_p"))
+		parameters := raw["parameters"].(map[string]any)
+		Expect(parameters).To(HaveKey("presence_penalty"))
+		Expect(parameters).To(HaveKey("repeat_penalty"))
+		Expect(parameters).To(HaveKey("min_p"))
+	})
+})

core/gallery/models.go

@@ -270,36 +270,47 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
 		lconfig.ApplyInferenceDefaults(&modelConfig, name, modelConfig.Model)
 
 		// Merge inference defaults into configMap so they are persisted without losing unknown fields.
+		// These sampling parameters live under the `parameters:` key on disk: ModelConfig
+		// embeds schema.PredictionOptions with `yaml:"parameters"`, so the loader only reads
+		// them from that submap. Writing them at the top level produced keys the loader never
+		// read back, leaving the persisted defaults inert on reload (#11230).
+		params, ok := configMap["parameters"].(map[string]any)
+		if !ok {
+			params = make(map[string]any)
+		}
 		if modelConfig.Temperature != nil {
-			if _, exists := configMap["temperature"]; !exists {
-				configMap["temperature"] = *modelConfig.Temperature
+			if _, exists := params["temperature"]; !exists {
+				params["temperature"] = *modelConfig.Temperature
 			}
 		}
 		if modelConfig.TopP != nil {
-			if _, exists := configMap["top_p"]; !exists {
-				configMap["top_p"] = *modelConfig.TopP
+			if _, exists := params["top_p"]; !exists {
+				params["top_p"] = *modelConfig.TopP
 			}
 		}
 		if modelConfig.TopK != nil {
-			if _, exists := configMap["top_k"]; !exists {
-				configMap["top_k"] = *modelConfig.TopK
+			if _, exists := params["top_k"]; !exists {
+				params["top_k"] = *modelConfig.TopK
 			}
 		}
 		if modelConfig.MinP != nil {
-			if _, exists := configMap["min_p"]; !exists {
-				configMap["min_p"] = *modelConfig.MinP
+			if _, exists := params["min_p"]; !exists {
+				params["min_p"] = *modelConfig.MinP
 			}
 		}
 		if modelConfig.RepeatPenalty != 0 {
-			if _, exists := configMap["repeat_penalty"]; !exists {
-				configMap["repeat_penalty"] = modelConfig.RepeatPenalty
+			if _, exists := params["repeat_penalty"]; !exists {
+				params["repeat_penalty"] = modelConfig.RepeatPenalty
 			}
 		}
 		if modelConfig.PresencePenalty != 0 {
-			if _, exists := configMap["presence_penalty"]; !exists {
-				configMap["presence_penalty"] = modelConfig.PresencePenalty
+			if _, exists := params["presence_penalty"]; !exists {
+				params["presence_penalty"] = modelConfig.PresencePenalty
 			}
 		}
+		if len(params) > 0 {
+			configMap["parameters"] = params
+		}
 
 		// Re-marshal from configMap to preserve unknown fields
 		updatedConfigYAML, err = yaml.Marshal(configMap)
@@ -494,4 +505,4 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
 		}
 	}
 	return nil
-}
+}