From b743f480b3d5a69d385d61e642ce2515ecb76969 Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Fri, 31 Jul 2026 09:16:59 -0700 Subject: [PATCH] fix(gallery): write installed inference defaults under parameters so the loader reads them back (#11230) ModelConfig embeds schema.PredictionOptions with `yaml:"parameters"`, so the loader only reads temperature/top_p/top_k/min_p/repeat_penalty/ presence_penalty from the parameters: submap. The gallery installer wrote those family defaults at the top level of the model YAML, where nothing reads them back, leaving every persisted default inert on reload. Merge them into the parameters: submap instead (preserving any values the config already sets there), and add a regression test that installs a qwen3.5 model and asserts the defaults round-trip through the typed loader rather than landing as inert top-level keys. The test is fully offline: the definition declares no files, so no download is attempted. Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Anai-Guo --- ...model_artifacts_inference_defaults_test.go | 63 +++++++++++++++++++ core/gallery/models.go | 37 +++++++---- 2 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 core/gallery/model_artifacts_inference_defaults_test.go diff --git a/core/gallery/model_artifacts_inference_defaults_test.go b/core/gallery/model_artifacts_inference_defaults_test.go new file mode 100644 index 000000000000..f19d9d97fcd8 --- /dev/null +++ b/core/gallery/model_artifacts_inference_defaults_test.go @@ -0,0 +1,63 @@ +package gallery_test + +import ( + "context" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("gallery inference-default persistence", func() { + It("persists inference defaults under parameters so the loader reads them back", func() { + modelsPath, err := os.MkdirTemp("", "inference-defaults") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(modelsPath) + + systemState, err := system.GetSystemState(system.WithModelPath(modelsPath)) + Expect(err).ToNot(HaveOccurred()) + + // A qwen3.5 name makes ApplyInferenceDefaults fill in the recommended + // sampling parameters (repeat_penalty=1, presence_penalty=1.5, min_p=0). + // Those belong under the parameters: key — ModelConfig embeds + // schema.PredictionOptions with `yaml:"parameters"`, so the loader only + // reads them from that submap (#11230). The install is fully offline: + // the definition declares no files, so InstallModel just writes the YAML. + definition := &ModelConfig{ConfigFile: `backend: transformers +parameters: + model: owner/repo +`} + + _, err = InstallModel(context.TODO(), systemState, "qwen3.5-managed", definition, map[string]any{}, func(string, string, string, float64) {}, false) + Expect(err).ToNot(HaveOccurred()) + + data, err := os.ReadFile(filepath.Join(modelsPath, "qwen3.5-managed.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // The defaults must survive a round-trip through the typed loader. + var reloaded config.ModelConfig + Expect(yaml.Unmarshal(data, &reloaded)).To(Succeed()) + Expect(reloaded.PresencePenalty).To(BeNumerically("==", 1.5)) + Expect(reloaded.RepeatPenalty).To(BeNumerically("==", 1)) + Expect(reloaded.MinP).NotTo(BeNil()) + Expect(reloaded.Temperature).NotTo(BeNil()) + + // They must live under parameters:, never at the top level, or they are + // silently dropped on reload. + var raw map[string]any + Expect(yaml.Unmarshal(data, &raw)).To(Succeed()) + Expect(raw).NotTo(HaveKey("presence_penalty")) + Expect(raw).NotTo(HaveKey("repeat_penalty")) + Expect(raw).NotTo(HaveKey("min_p")) + parameters, ok := raw["parameters"].(map[string]any) + Expect(ok).To(BeTrue()) + Expect(parameters).To(HaveKey("presence_penalty")) + Expect(parameters).To(HaveKey("repeat_penalty")) + Expect(parameters).To(HaveKey("min_p")) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 82a36d7a5d8b..0a84462e676e 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -270,36 +270,47 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver lconfig.ApplyInferenceDefaults(&modelConfig, name, modelConfig.Model) // Merge inference defaults into configMap so they are persisted without losing unknown fields. + // These sampling parameters live under the `parameters:` key on disk: ModelConfig + // embeds schema.PredictionOptions with `yaml:"parameters"`, so the loader only reads + // them from that submap. Writing them at the top level produced keys the loader never + // read back, leaving the persisted defaults inert on reload (#11230). + params, ok := configMap["parameters"].(map[string]any) + if !ok { + params = make(map[string]any) + } if modelConfig.Temperature != nil { - if _, exists := configMap["temperature"]; !exists { - configMap["temperature"] = *modelConfig.Temperature + if _, exists := params["temperature"]; !exists { + params["temperature"] = *modelConfig.Temperature } } if modelConfig.TopP != nil { - if _, exists := configMap["top_p"]; !exists { - configMap["top_p"] = *modelConfig.TopP + if _, exists := params["top_p"]; !exists { + params["top_p"] = *modelConfig.TopP } } if modelConfig.TopK != nil { - if _, exists := configMap["top_k"]; !exists { - configMap["top_k"] = *modelConfig.TopK + if _, exists := params["top_k"]; !exists { + params["top_k"] = *modelConfig.TopK } } if modelConfig.MinP != nil { - if _, exists := configMap["min_p"]; !exists { - configMap["min_p"] = *modelConfig.MinP + if _, exists := params["min_p"]; !exists { + params["min_p"] = *modelConfig.MinP } } if modelConfig.RepeatPenalty != 0 { - if _, exists := configMap["repeat_penalty"]; !exists { - configMap["repeat_penalty"] = modelConfig.RepeatPenalty + if _, exists := params["repeat_penalty"]; !exists { + params["repeat_penalty"] = modelConfig.RepeatPenalty } } if modelConfig.PresencePenalty != 0 { - if _, exists := configMap["presence_penalty"]; !exists { - configMap["presence_penalty"] = modelConfig.PresencePenalty + if _, exists := params["presence_penalty"]; !exists { + params["presence_penalty"] = modelConfig.PresencePenalty } } + if len(params) > 0 { + configMap["parameters"] = params + } // Re-marshal from configMap to preserve unknown fields updatedConfigYAML, err = yaml.Marshal(configMap) @@ -494,4 +505,4 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error { } } return nil -} +} \ No newline at end of file