Skip to content

Commit 5c35e85

Browse files
authored
feat: allow to pin models and skip from reaping (#9309)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 062e0d0 commit 5c35e85

10 files changed

Lines changed: 366 additions & 2 deletions

File tree

core/application/watchdog.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,27 @@ import (
55
"github.com/mudler/xlog"
66
)
77

8+
// SyncPinnedModelsToWatchdog reads pinned status from all model configs and updates the watchdog
9+
func (a *Application) SyncPinnedModelsToWatchdog() {
10+
cl := a.ModelConfigLoader()
11+
if cl == nil {
12+
return
13+
}
14+
wd := a.modelLoader.GetWatchDog()
15+
if wd == nil {
16+
return
17+
}
18+
configs := cl.GetAllModelsConfigs()
19+
var pinned []string
20+
for _, cfg := range configs {
21+
if cfg.IsPinned() {
22+
pinned = append(pinned, cfg.Name)
23+
}
24+
}
25+
wd.SetPinnedModels(pinned)
26+
xlog.Debug("Synced pinned models to watchdog", "count", len(pinned))
27+
}
28+
829
func (a *Application) StopWatchdog() error {
930
if a.watchdogStop != nil {
1031
close(a.watchdogStop)
@@ -44,6 +65,9 @@ func (a *Application) startWatchdog() error {
4465
// Set the watchdog on the model loader
4566
a.modelLoader.SetWatchDog(wd)
4667

68+
// Sync pinned models from config to the watchdog
69+
a.SyncPinnedModelsToWatchdog()
70+
4771
// Start watchdog goroutine if any periodic checks are enabled
4872
// LRU eviction doesn't need the Run() loop - it's triggered on model load
4973
// But memory reclaimer needs the Run() loop for periodic checking
@@ -124,5 +148,8 @@ func (a *Application) RestartWatchdog() error {
124148
newWD.RestoreState(oldState)
125149
}
126150

151+
// Re-sync pinned models after restart
152+
a.SyncPinnedModelsToWatchdog()
153+
127154
return nil
128155
}

core/config/model_config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ type ModelConfig struct {
7878
Description string `yaml:"description,omitempty" json:"description,omitempty"`
7979
Usage string `yaml:"usage,omitempty" json:"usage,omitempty"`
8080
Disabled *bool `yaml:"disabled,omitempty" json:"disabled,omitempty"`
81+
Pinned *bool `yaml:"pinned,omitempty" json:"pinned,omitempty"`
8182

8283
Options []string `yaml:"options,omitempty" json:"options,omitempty"`
8384
Overrides []string `yaml:"overrides,omitempty" json:"overrides,omitempty"`
@@ -554,6 +555,11 @@ func (c *ModelConfig) IsDisabled() bool {
554555
return c.Disabled != nil && *c.Disabled
555556
}
556557

558+
// IsPinned returns true if the model is pinned (excluded from idle unloading and eviction)
559+
func (c *ModelConfig) IsPinned() bool {
560+
return c.Pinned != nil && *c.Pinned
561+
}
562+
557563
type ModelConfigUsecase int
558564

559565
const (
Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package localai
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"net/url"
7+
"os"
8+
9+
"github.com/labstack/echo/v4"
10+
"github.com/mudler/LocalAI/core/config"
11+
"github.com/mudler/LocalAI/pkg/utils"
12+
13+
"gopkg.in/yaml.v3"
14+
)
15+
16+
// TogglePinnedModelEndpoint handles pinning or unpinning a model.
17+
// Pinned models are excluded from idle unloading, LRU eviction, and memory-pressure eviction.
18+
//
19+
// @Summary Toggle model pinned status
20+
// @Description Pin or unpin a model. Pinned models stay loaded and are excluded from automatic eviction.
21+
// @Tags config
22+
// @Param name path string true "Model name"
23+
// @Param action path string true "Action: 'pin' or 'unpin'"
24+
// @Success 200 {object} ModelResponse
25+
// @Failure 400 {object} ModelResponse
26+
// @Failure 404 {object} ModelResponse
27+
// @Failure 500 {object} ModelResponse
28+
// @Router /api/models/toggle-pinned/{name}/{action} [put]
29+
func TogglePinnedModelEndpoint(cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, syncPinnedFn func()) echo.HandlerFunc {
30+
return func(c echo.Context) error {
31+
modelName := c.Param("name")
32+
if decoded, err := url.PathUnescape(modelName); err == nil {
33+
modelName = decoded
34+
}
35+
if modelName == "" {
36+
return c.JSON(http.StatusBadRequest, ModelResponse{
37+
Success: false,
38+
Error: "Model name is required",
39+
})
40+
}
41+
42+
action := c.Param("action")
43+
if action != "pin" && action != "unpin" {
44+
return c.JSON(http.StatusBadRequest, ModelResponse{
45+
Success: false,
46+
Error: "Action must be 'pin' or 'unpin'",
47+
})
48+
}
49+
50+
// Get existing model config
51+
modelConfig, exists := cl.GetModelConfig(modelName)
52+
if !exists {
53+
return c.JSON(http.StatusNotFound, ModelResponse{
54+
Success: false,
55+
Error: "Model configuration not found",
56+
})
57+
}
58+
59+
// Get the config file path
60+
configPath := modelConfig.GetModelConfigFile()
61+
if configPath == "" {
62+
return c.JSON(http.StatusNotFound, ModelResponse{
63+
Success: false,
64+
Error: "Model configuration file not found",
65+
})
66+
}
67+
68+
// Verify the path is trusted
69+
if err := utils.VerifyPath(configPath, appConfig.SystemState.Model.ModelsPath); err != nil {
70+
return c.JSON(http.StatusForbidden, ModelResponse{
71+
Success: false,
72+
Error: "Model configuration not trusted: " + err.Error(),
73+
})
74+
}
75+
76+
// Read the existing config file
77+
configData, err := os.ReadFile(configPath)
78+
if err != nil {
79+
return c.JSON(http.StatusInternalServerError, ModelResponse{
80+
Success: false,
81+
Error: "Failed to read configuration file: " + err.Error(),
82+
})
83+
}
84+
85+
// Parse the YAML config as a generic map to preserve all fields
86+
var configMap map[string]interface{}
87+
if err := yaml.Unmarshal(configData, &configMap); err != nil {
88+
return c.JSON(http.StatusInternalServerError, ModelResponse{
89+
Success: false,
90+
Error: "Failed to parse configuration file: " + err.Error(),
91+
})
92+
}
93+
94+
// Update the pinned field
95+
pinned := action == "pin"
96+
if pinned {
97+
configMap["pinned"] = true
98+
} else {
99+
// Remove the pinned key entirely when unpinning (clean YAML)
100+
delete(configMap, "pinned")
101+
}
102+
103+
// Marshal back to YAML
104+
updatedData, err := yaml.Marshal(configMap)
105+
if err != nil {
106+
return c.JSON(http.StatusInternalServerError, ModelResponse{
107+
Success: false,
108+
Error: "Failed to serialize configuration: " + err.Error(),
109+
})
110+
}
111+
112+
// Write updated config back to file
113+
if err := os.WriteFile(configPath, updatedData, 0644); err != nil {
114+
return c.JSON(http.StatusInternalServerError, ModelResponse{
115+
Success: false,
116+
Error: "Failed to write configuration file: " + err.Error(),
117+
})
118+
}
119+
120+
// Reload model configurations from disk
121+
if err := cl.LoadModelConfigsFromPath(appConfig.SystemState.Model.ModelsPath, appConfig.ToConfigLoaderOptions()...); err != nil {
122+
return c.JSON(http.StatusInternalServerError, ModelResponse{
123+
Success: false,
124+
Error: "Failed to reload configurations: " + err.Error(),
125+
})
126+
}
127+
128+
// Sync pinned models to the watchdog
129+
if syncPinnedFn != nil {
130+
syncPinnedFn()
131+
}
132+
133+
msg := fmt.Sprintf("Model '%s' has been %sned successfully.", modelName, action)
134+
if pinned {
135+
msg += " The model will be excluded from automatic eviction."
136+
}
137+
138+
return c.JSON(http.StatusOK, ModelResponse{
139+
Success: true,
140+
Message: msg,
141+
Filename: configPath,
142+
})
143+
}
144+
}

core/http/react-ui/src/pages/Manage.jsx

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ export default function Manage() {
2525
const [confirmDialog, setConfirmDialog] = useState(null)
2626
const [distributedMode, setDistributedMode] = useState(false)
2727
const [togglingModels, setTogglingModels] = useState(new Set())
28+
const [pinningModels, setPinningModels] = useState(new Set())
2829

2930
const handleTabChange = (tab) => {
3031
setActiveTab(tab)
@@ -122,6 +123,24 @@ export default function Manage() {
122123
}
123124
}
124125

126+
const handleTogglePinned = async (modelId, currentlyPinned) => {
127+
const action = currentlyPinned ? 'unpin' : 'pin'
128+
setPinningModels(prev => new Set(prev).add(modelId))
129+
try {
130+
await modelsApi.togglePinned(modelId, action)
131+
addToast(`Model ${modelId} ${action}ned`, 'success')
132+
refetchModels()
133+
} catch (err) {
134+
addToast(`Failed to ${action} model: ${err.message}`, 'error')
135+
} finally {
136+
setPinningModels(prev => {
137+
const next = new Set(prev)
138+
next.delete(modelId)
139+
return next
140+
})
141+
}
142+
}
143+
125144
const handleReload = async () => {
126145
setReloading(true)
127146
try {
@@ -303,6 +322,22 @@ export default function Manage() {
303322
<i className="fas fa-stop" />
304323
</button>
305324
)}
325+
{/* Pin button - prevents model from being unloaded */}
326+
<button
327+
className="btn btn-sm"
328+
onClick={() => handleTogglePinned(model.id, model.pinned)}
329+
disabled={pinningModels.has(model.id) || model.disabled}
330+
title={model.pinned ? 'Unpin model (allow idle unloading)' : 'Pin model (prevent idle unloading)'}
331+
style={{
332+
padding: '2px 6px',
333+
minWidth: 28,
334+
color: model.pinned ? 'var(--color-warning, #f59e0b)' : 'var(--color-text-muted)',
335+
opacity: model.disabled ? 0.3 : (pinningModels.has(model.id) ? 0.5 : 1),
336+
cursor: pinningModels.has(model.id) ? 'wait' : (model.disabled ? 'not-allowed' : 'pointer'),
337+
}}
338+
>
339+
<i className={`fas fa-thumbtack${pinningModels.has(model.id) ? ' fa-spin' : ''}`} />
340+
</button>
306341
{/* Toggle switch for enabling/disabling model loading on demand */}
307342
<label
308343
title={model.disabled ? 'Model is disabled — click to enable loading on demand' : 'Model is enabled — click to disable loading on demand'}

core/http/react-ui/src/utils/api.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ export const modelsApi = {
9898
getEditConfig: (name) => fetchJSON(API_CONFIG.endpoints.modelEditGet(name)),
9999
editConfig: (name, body) => postJSON(API_CONFIG.endpoints.modelEdit(name), body),
100100
toggleState: (name, action) => fetchJSON(API_CONFIG.endpoints.modelToggleState(name, action), { method: 'PUT' }),
101+
togglePinned: (name, action) => fetchJSON(API_CONFIG.endpoints.modelTogglePinned(name, action), { method: 'PUT' }),
101102
getConfigMetadata: (section) => fetchJSON(
102103
section ? `${API_CONFIG.endpoints.configMetadata}?section=${section}`
103104
: API_CONFIG.endpoints.configMetadata

core/http/react-ui/src/utils/config.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ export const API_CONFIG = {
9494
modelEditGet: (name) => `/api/models/edit/${name}`,
9595
modelEdit: (name) => `/models/edit/${name}`,
9696
modelToggleState: (name, action) => `/models/toggle-state/${name}/${action}`,
97+
modelTogglePinned: (name, action) => `/models/toggle-pinned/${name}/${action}`,
9798
backendsAvailable: '/backends/available',
9899
backendsInstalled: '/backends',
99100
version: '/version',

core/http/routes/localai.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ func RegisterLocalAIRoutes(router *echo.Echo,
7878
// Toggle model enable/disable endpoint
7979
router.PUT("/models/toggle-state/:name/:action", localai.ToggleStateModelEndpoint(cl, ml, appConfig), adminMiddleware)
8080

81+
// Toggle model pinned status endpoint
82+
router.PUT("/models/toggle-pinned/:name/:action", localai.TogglePinnedModelEndpoint(cl, appConfig, func() {
83+
app.SyncPinnedModelsToWatchdog()
84+
}), adminMiddleware)
85+
8186
// Reload models endpoint
8287
router.POST("/models/reload", localai.ReloadModelsEndpoint(cl, appConfig), adminMiddleware)
8388
}

core/http/routes/ui_api.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
515515
Capabilities []string `json:"capabilities"`
516516
Backend string `json:"backend"`
517517
Disabled bool `json:"disabled"`
518+
Pinned bool `json:"pinned"`
518519
}
519520

520521
result := make([]modelCapability, 0, len(modelConfigs)+len(modelsWithoutConfig))
@@ -524,6 +525,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
524525
Capabilities: cfg.KnownUsecaseStrings,
525526
Backend: cfg.Backend,
526527
Disabled: cfg.IsDisabled(),
528+
Pinned: cfg.IsPinned(),
527529
})
528530
}
529531
for _, name := range modelsWithoutConfig {

pkg/model/watchdog.go

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ type WatchDog struct {
4545

4646
// Eviction settings
4747
forceEvictionWhenBusy bool // Force eviction even when models have active API calls (default: false for safety)
48+
49+
// Pinned models are excluded from idle, LRU, and memory-pressure eviction
50+
pinnedModels map[string]bool
4851
}
4952

5053
type ProcessManager interface {
@@ -78,6 +81,7 @@ func NewWatchDog(opts ...WatchDogOption) *WatchDog {
7881
idleCheck: o.idleCheck,
7982
lruLimit: o.lruLimit,
8083
addressModelMap: make(map[string]string),
84+
pinnedModels: make(map[string]bool),
8185
stop: make(chan bool, 1),
8286
done: make(chan bool, 1),
8387
memoryReclaimerEnabled: o.memoryReclaimerEnabled,
@@ -123,6 +127,24 @@ func (wd *WatchDog) SetForceEvictionWhenBusy(force bool) {
123127
wd.forceEvictionWhenBusy = force
124128
}
125129

130+
// SetPinnedModels replaces the set of pinned model names.
131+
// Pinned models are excluded from idle, LRU, and memory-pressure eviction.
132+
func (wd *WatchDog) SetPinnedModels(models []string) {
133+
wd.Lock()
134+
defer wd.Unlock()
135+
wd.pinnedModels = make(map[string]bool, len(models))
136+
for _, m := range models {
137+
wd.pinnedModels[m] = true
138+
}
139+
}
140+
141+
// IsModelPinned returns true if the given model name is pinned
142+
func (wd *WatchDog) IsModelPinned(modelName string) bool {
143+
wd.Lock()
144+
defer wd.Unlock()
145+
return wd.pinnedModels[modelName]
146+
}
147+
126148
func (wd *WatchDog) Shutdown() {
127149
wd.Lock()
128150
defer wd.Unlock()
@@ -310,6 +332,11 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) EnforceLRULimitResult {
310332
skippedBusyCount := 0
311333
for i := 0; evictedCount < modelsToEvict && i < len(models); i++ {
312334
m := models[i]
335+
// Skip pinned models
336+
if wd.pinnedModels[m.model] {
337+
xlog.Debug("[WatchDog] Skipping LRU eviction for pinned model", "model", m.model)
338+
continue
339+
}
313340
// Check if model is busy
314341
_, isBusy := wd.busyTime[m.address]
315342
if isBusy && !forceEvictionWhenBusy {
@@ -389,9 +416,13 @@ func (wd *WatchDog) checkIdle() {
389416
for address, t := range wd.idleTime {
390417
xlog.Debug("[WatchDog] idle connection", "address", address)
391418
if time.Since(t) > wd.idletimeout {
392-
xlog.Warn("[WatchDog] Address is idle for too long, killing it", "address", address)
393419
model, ok := wd.addressModelMap[address]
394420
if ok {
421+
if wd.pinnedModels[model] {
422+
xlog.Debug("[WatchDog] Skipping idle eviction for pinned model", "model", model)
423+
continue
424+
}
425+
xlog.Warn("[WatchDog] Address is idle for too long, killing it", "address", address)
395426
modelsToShutdown = append(modelsToShutdown, model)
396427
} else {
397428
xlog.Warn("[WatchDog] Address unresolvable", "address", address)
@@ -514,10 +545,14 @@ func (wd *WatchDog) evictLRUModel() {
514545
return a.lastUsed.Compare(b.lastUsed)
515546
})
516547

517-
// Find the first non-busy model (or first model if forceEvictionWhenBusy is true)
548+
// Find the first non-busy, non-pinned model (or first non-pinned model if forceEvictionWhenBusy is true)
518549
var lruModel *modelUsageInfo
519550
for i := range len(models) {
520551
m := models[i]
552+
if wd.pinnedModels[m.model] {
553+
xlog.Debug("[WatchDog] Skipping memory reclaimer eviction for pinned model", "model", m.model)
554+
continue
555+
}
521556
_, isBusy := wd.busyTime[m.address]
522557
if isBusy && !forceEvictionWhenBusy {
523558
// Skip busy models when forceEvictionWhenBusy is false

0 commit comments

Comments
 (0)