feat: inferencing default, automatic tool parsing fallback and wire min_p

.github/workflows/bump-inference-defaults.yml

@@ -0,0 +1,48 @@
+name: Bump inference defaults
+
+on:
+  schedule:
+    # Run daily at 06:00 UTC
+    - cron: '0 6 * * *'
+  workflow_dispatch: # Allow manual trigger
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Re-fetch inference defaults
+        run: make generate-force
+
+      - name: Check for changes
+        id: diff
+        run: |
+          if git diff --quiet core/config/inference_defaults.json; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create Pull Request
+        if: steps.diff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: "chore: bump inference defaults from unsloth"
+          title: "chore: bump inference defaults from unsloth"
+          body: |
+            Auto-generated update of `core/config/inference_defaults.json` from
+            [unsloth's inference_defaults.json](https://github.com/unslothai/unsloth/blob/main/studio/backend/assets/configs/inference_defaults.json).
+
+            This PR was created automatically by the `bump-inference-defaults` workflow.
+          branch: chore/bump-inference-defaults
+          delete-branch: true
+          labels: automated

Makefile

@@ -107,7 +107,7 @@ core/http/react-ui/dist: react-ui
 
 ## Build:
 
-build: protogen-go install-go-tools core/http/react-ui/dist ## Build the project
+build: protogen-go generate install-go-tools core/http/react-ui/dist ## Build the project
 	$(info ${GREEN}I local-ai build info:${RESET})
 	$(info ${GREEN}I BUILD_TYPE: ${YELLOW}$(BUILD_TYPE)${RESET})
 	$(info ${GREEN}I GO_TAGS: ${YELLOW}$(GO_TAGS)${RESET})
@@ -398,6 +398,16 @@ protogen-go: protoc install-go-tools
 	./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
     backend/backend.proto
 
+core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing)
+	$(GOCMD) generate ./core/config/...
+
+.PHONY: generate
+generate: core/config/inference_defaults.json ## Ensure inference defaults exist
+
+.PHONY: generate-force
+generate-force: ## Re-fetch inference defaults from unsloth (always)
+	$(GOCMD) generate ./core/config/...
+
 .PHONY: protogen-go-clean
 protogen-go-clean:
 	$(RM) pkg/grpc/proto/backend.pb.go pkg/grpc/proto/backend_grpc.pb.go

backend/backend.proto

@@ -173,6 +173,7 @@ message PredictOptions {
   int32 Logprobs = 50;  // Number of top logprobs to return (maps to OpenAI logprobs parameter)
   int32 TopLogprobs = 51;  // Number of top logprobs to return per token (maps to OpenAI top_logprobs parameter)
   map<string, string> Metadata = 52;  // Generic per-request metadata (e.g., enable_thinking)
+  float MinP = 53;  // Minimum probability sampling threshold (0.0 = disabled)
 }
 
 // ToolCallDelta represents an incremental tool call update from the C++ parser.

backend/cpp/llama-cpp/grpc-server.cpp

@@ -136,6 +136,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
     data["mirostat_eta"] = predict->mirostateta();
     data["n_keep"] = predict->nkeep();
     data["seed"] = predict->seed();
+    data["min_p"] = predict->minp();
 
 
     std::string grammar_str = predict->grammar();

core/backend/options.go

@@ -252,6 +252,7 @@ func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions
 		TopP:                float32(*c.TopP),
 		NDraft:              c.NDraft,
 		TopK:                int32(*c.TopK),
+		MinP:                float32(*c.MinP),
 		Tokens:              int32(*c.Maxtokens),
 		Threads:             int32(*c.Threads),
 		PromptCacheAll:      c.PromptCacheAll,

core/config/gen_inference_defaults/README.md

@@ -0,0 +1,30 @@
+# gen_inference_defaults
+
+This tool fetches per-model-family inference parameter defaults from [unsloth's inference_defaults.json](https://github.com/unslothai/unsloth/blob/main/studio/backend/assets/configs/inference_defaults.json), validates the data, remaps field names to LocalAI conventions, and writes `core/config/inference_defaults.json`.
+
+## What it does
+
+1. Fetches the latest `inference_defaults.json` from unsloth's repo
+2. Validates that every entry has required fields (`temperature`, `top_p`, `top_k`)
+3. Validates that every pattern references an existing family
+4. Warns if pattern ordering would cause shorter prefixes to shadow longer ones
+5. Remaps `repetition_penalty` → `repeat_penalty` (LocalAI naming)
+6. Filters to allowed fields only: `temperature`, `top_p`, `top_k`, `min_p`, `repeat_penalty`, `presence_penalty`
+7. Writes the validated JSON to `core/config/inference_defaults.json`
+
+## Usage
+
+```bash
+# Only regenerate if the file is missing (runs during make build)
+make generate
+
+# Force re-fetch from unsloth
+make generate-force
+
+# Or directly via go generate
+go generate ./core/config/...
+```
+
+## Automation
+
+The GitHub Actions workflow `.github/workflows/bump-inference-defaults.yml` runs `make generate-force` daily and opens a PR if the upstream data changed.

core/config/gen_inference_defaults/main.go

@@ -0,0 +1,222 @@
+// gen_inference_defaults fetches unsloth's inference_defaults.json,
+// validates its structure, remaps field names to LocalAI conventions,
+// and writes the result to core/config/inference_defaults.json.
+//
+// Run via: go generate ./core/config/
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+)
+
+const (
+	unslothURL = "https://raw.githubusercontent.com/unslothai/unsloth/main/studio/backend/assets/configs/inference_defaults.json"
+	outputFile = "inference_defaults.json"
+)
+
+// unslothDefaults mirrors the upstream JSON structure
+type unslothDefaults struct {
+	Comment  string                        `json:"_comment"`
+	Families map[string]map[string]float64 `json:"families"`
+	Patterns []string                      `json:"patterns"`
+}
+
+// localAIDefaults is our output structure
+type localAIDefaults struct {
+	Comment  string                        `json:"_comment"`
+	Families map[string]map[string]float64 `json:"families"`
+	Patterns []string                      `json:"patterns"`
+}
+
+// requiredFields are the fields every family entry must have
+var requiredFields = []string{"temperature", "top_p", "top_k"}
+
+// fieldRemap maps unsloth field names to LocalAI field names
+var fieldRemap = map[string]string{
+	"repetition_penalty": "repeat_penalty",
+}
+
+// allowedFields are the only fields we keep (after remapping)
+var allowedFields = map[string]bool{
+	"temperature":      true,
+	"top_p":            true,
+	"top_k":            true,
+	"min_p":            true,
+	"repeat_penalty":   true,
+	"presence_penalty": true,
+}
+
+func main() {
+	fmt.Fprintf(os.Stderr, "Fetching %s ...\n", unslothURL)
+
+	resp, err := http.Get(unslothURL)
+	if err != nil {
+		fatal("fetch failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		fatal("fetch returned HTTP %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		fatal("read body: %v", err)
+	}
+
+	var upstream unslothDefaults
+	if err := json.Unmarshal(body, &upstream); err != nil {
+		fatal("parse upstream JSON: %v", err)
+	}
+
+	// Validate structure
+	if len(upstream.Families) == 0 {
+		fatal("upstream has no families")
+	}
+	if len(upstream.Patterns) == 0 {
+		fatal("upstream has no patterns")
+	}
+
+	// Validate every pattern references a family
+	for _, p := range upstream.Patterns {
+		if _, ok := upstream.Families[p]; !ok {
+			fatal("pattern %q has no corresponding family entry", p)
+		}
+	}
+
+	// Validate every family has required fields and remap field names
+	output := localAIDefaults{
+		Comment:  "Auto-generated from unsloth inference_defaults.json. DO NOT EDIT. Run go generate ./core/config/ to update.",
+		Families: make(map[string]map[string]float64, len(upstream.Families)),
+		Patterns: upstream.Patterns,
+	}
+
+	// Sort family names for deterministic output
+	familyNames := make([]string, 0, len(upstream.Families))
+	for name := range upstream.Families {
+		familyNames = append(familyNames, name)
+	}
+	sort.Strings(familyNames)
+
+	for _, name := range familyNames {
+		params := upstream.Families[name]
+
+		// Check required fields
+		for _, req := range requiredFields {
+			found := false
+			for k := range params {
+				mapped := k
+				if m, ok := fieldRemap[k]; ok {
+					mapped = m
+				}
+				if mapped == req || k == req {
+					found = true
+					break
+				}
+			}
+			if !found {
+				fatal("family %q missing required field %q", name, req)
+			}
+		}
+
+		// Remap and filter fields
+		remapped := make(map[string]float64)
+		for k, v := range params {
+			if newName, ok := fieldRemap[k]; ok {
+				k = newName
+			}
+			if allowedFields[k] {
+				remapped[k] = v
+			}
+		}
+		output.Families[name] = remapped
+	}
+
+	// Validate patterns are ordered longest-match-first within same prefix groups
+	validatePatternOrder(output.Patterns)
+
+	// Marshal with ordered keys for readability
+	data, err := marshalOrdered(output)
+	if err != nil {
+		fatal("marshal output: %v", err)
+	}
+
+	if err := os.WriteFile(outputFile, data, 0644); err != nil {
+		fatal("write %s: %v", outputFile, err)
+	}
+
+	fmt.Fprintf(os.Stderr, "Written %s (%d families, %d patterns)\n",
+		outputFile, len(output.Families), len(output.Patterns))
+}
+
+// validatePatternOrder warns if a shorter pattern appears before a longer one
+// that it's a prefix of (e.g., "qwen3" before "qwen3.5")
+func validatePatternOrder(patterns []string) {
+	for i, p := range patterns {
+		for j := i + 1; j < len(patterns); j++ {
+			if strings.HasPrefix(patterns[j], p) {
+				fmt.Fprintf(os.Stderr, "WARNING: pattern %q at index %d is a prefix of %q at index %d — longer match should come first\n",
+					p, i, patterns[j], j)
+			}
+		}
+	}
+}
+
+// marshalOrdered produces JSON with families in pattern order for readability
+func marshalOrdered(d localAIDefaults) ([]byte, error) {
+	var sb strings.Builder
+	sb.WriteString("{\n")
+	sb.WriteString(fmt.Sprintf("  %q: %q,\n", "_comment", d.Comment))
+	sb.WriteString("  \"families\": {\n")
+
+	// Write families in pattern order, then any remaining not in patterns
+	written := make(map[string]bool)
+	allFamilies := make([]string, 0, len(d.Families))
+	for _, p := range d.Patterns {
+		if _, ok := d.Families[p]; ok && !written[p] {
+			allFamilies = append(allFamilies, p)
+			written[p] = true
+		}
+	}
+	for name := range d.Families {
+		if !written[name] {
+			allFamilies = append(allFamilies, name)
+		}
+	}
+
+	for i, name := range allFamilies {
+		params := d.Families[name]
+		paramJSON, err := json.Marshal(params)
+		if err != nil {
+			return nil, err
+		}
+		comma := ","
+		if i == len(allFamilies)-1 {
+			comma = ""
+		}
+		sb.WriteString(fmt.Sprintf("    %q: %s%s\n", name, paramJSON, comma))
+	}
+
+	sb.WriteString("  },\n")
+
+	// Patterns array
+	patternsJSON, err := json.Marshal(d.Patterns)
+	if err != nil {
+		return nil, err
+	}
+	sb.WriteString(fmt.Sprintf("  \"patterns\": %s\n", patternsJSON))
+	sb.WriteString("}\n")
+
+	return []byte(sb.String()), nil
+}
+
+func fatal(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, "gen_inference_defaults: "+format+"\n", args...)
+	os.Exit(1)
+}

core/config/gguf.go

@@ -76,6 +76,8 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
 	cfg.Options = append(cfg.Options, "use_jinja:true")
 	cfg.KnownUsecaseStrings = append(cfg.KnownUsecaseStrings, "FLAG_CHAT")
 
+	// Apply per-model-family inference parameter defaults (temperature, top_p, etc.)
+	ApplyInferenceDefaults(cfg, f.Metadata().Name)
 }
 
 // DetectThinkingSupportFromBackend calls the ModelMetadata gRPC method to detect