Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions core/config/application_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,16 @@ func (o *ApplicationConfig) GetEffectiveMaxActiveBackends() int {
return 0
}

// WatchdogShouldRun reports whether the live watchdog process should be
// running for the current config. It mirrors the gating in
// (*Application).startWatchdog so the /api/settings start/stop decision and
// the startup path agree on a single source of truth: the watchdog runs when
// idle/busy checks are enabled (WatchDog), when LRU eviction is active
// (effective max active backends > 0), or when the memory reclaimer is on.
func (o *ApplicationConfig) WatchdogShouldRun() bool {
return o.WatchDog || o.GetEffectiveMaxActiveBackends() > 0 || o.MemoryReclaimerEnabled
}

// WithForceEvictionWhenBusy sets whether to force eviction even when models have active API calls
func WithForceEvictionWhenBusy(enabled bool) AppOption {
return func(o *ApplicationConfig) {
Expand Down Expand Up @@ -1198,18 +1208,22 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req
}
if settings.WatchdogIdleEnabled != nil {
o.WatchDogIdle = *settings.WatchdogIdleEnabled
if o.WatchDogIdle {
o.WatchDog = true
}
requireRestart = true
}
if settings.WatchdogBusyEnabled != nil {
o.WatchDogBusy = *settings.WatchdogBusyEnabled
if o.WatchDogBusy {
o.WatchDog = true
}
requireRestart = true
}
// The React Settings "Enable Watchdog" master toggle manages only the
// idle/busy checks — watchdog_enabled is vestigial in that UI. Whenever
// either idle/busy field is present in the body, derive the run-state from
// idle||busy so a cold enable starts the watchdog and a full disable stops
// it, instead of trusting the stale watchdog_enabled the UI never updates.
// This mirrors the startup invariant in startup.go. An API client posting
// only watchdog_enabled (idle/busy absent) keeps its explicit value.
if settings.WatchdogIdleEnabled != nil || settings.WatchdogBusyEnabled != nil {
o.WatchDog = o.WatchDogIdle || o.WatchDogBusy
}
if settings.WatchdogIdleTimeout != nil {
if dur, err := time.ParseDuration(*settings.WatchdogIdleTimeout); err == nil {
o.WatchDogIdleTimeout = dur
Expand Down
63 changes: 63 additions & 0 deletions core/config/application_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,69 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
Expect(appConfig.WatchDogBusy).To(BeTrue())
})

// Residual #9125: the React Settings "Enable Watchdog" master toggle
// manages only watchdog_idle_enabled / watchdog_busy_enabled — it never
// touches the vestigial watchdog_enabled field. On a cold enable the
// body therefore carries watchdog_enabled=false alongside idle/busy=true.
// The derived run-state (WatchDog) must follow idle||busy so the live
// watchdog actually starts, not the stale watchdog_enabled=false.
It("should derive WatchDog from idle||busy on a cold enable even when watchdog_enabled=false", func() {
appConfig := &ApplicationConfig{WatchDog: false}

watchdogEnabled := false
watchdogIdle := true
watchdogBusy := true
rs := &RuntimeSettings{
WatchdogEnabled: &watchdogEnabled,
WatchdogIdleEnabled: &watchdogIdle,
WatchdogBusyEnabled: &watchdogBusy,
}

appConfig.ApplyRuntimeSettings(rs)

Expect(appConfig.WatchDog).To(BeTrue())
Expect(appConfig.WatchdogShouldRun()).To(BeTrue())
})

// The disable direction: the master toggle off sends idle=false,
// busy=false, but watchdog_enabled may still be the stale true loaded
// before the change. WatchDog must follow idle||busy down to false so
// the live watchdog is stopped (it stays stopped unless LRU / memory
// reclaimer keep it alive, which is gated by WatchdogShouldRun).
It("should disable WatchDog when both idle and busy are turned off", func() {
appConfig := &ApplicationConfig{WatchDog: true, WatchDogIdle: true, WatchDogBusy: true}

watchdogEnabled := true
watchdogIdle := false
watchdogBusy := false
rs := &RuntimeSettings{
WatchdogEnabled: &watchdogEnabled,
WatchdogIdleEnabled: &watchdogIdle,
WatchdogBusyEnabled: &watchdogBusy,
}

appConfig.ApplyRuntimeSettings(rs)

Expect(appConfig.WatchDog).To(BeFalse())
Expect(appConfig.WatchdogShouldRun()).To(BeFalse())
})

// Backward compatibility: an API client that posts only watchdog_enabled
// (idle/busy nil) keeps the explicit value — the idle/busy derivation
// only kicks in when those fields are actually present in the body.
It("should preserve explicit watchdog_enabled when idle/busy are absent", func() {
appConfig := &ApplicationConfig{WatchDog: false}

watchdogEnabled := true
rs := &RuntimeSettings{
WatchdogEnabled: &watchdogEnabled,
}

appConfig.ApplyRuntimeSettings(rs)

Expect(appConfig.WatchDog).To(BeTrue())
})

It("should handle MaxActiveBackends and update SingleBackend accordingly", func() {
appConfig := &ApplicationConfig{}

Expand Down
13 changes: 11 additions & 2 deletions core/http/endpoints/localai/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,9 +221,18 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
// Check if agent job retention changed
agentJobChanged := settings.AgentJobRetentionDays != nil

// Restart watchdog if settings changed
// Restart watchdog if settings changed.
//
// The live start/stop decision derives from the post-apply config
// (WatchdogShouldRun) rather than the raw watchdog_enabled request
// field: the React master toggle only ever writes the idle/busy flags,
// so keying off watchdog_enabled left the live watchdog stopped on a
// cold enable until the next restart (#9125). WatchdogShouldRun mirrors
// the gating in startWatchdog, so a cold enable starts it immediately
// and a full disable (both checks off, no LRU / memory reclaimer) stops
// it.
if watchdogChanged {
if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled {
if !appConfig.WatchdogShouldRun() {
if err := app.StopWatchdog(); err != nil {
xlog.Error("Failed to stop watchdog", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Expand Down
16 changes: 16 additions & 0 deletions core/http/endpoints/localai/settings_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,20 @@ var _ = Describe("Settings endpoints", func() {
_, err := os.Stat(filepath.Join(tmp, "runtime_settings.json"))
Expect(err).ToNot(HaveOccurred())
})

// Residual #9125: enabling the watchdog from a cold (off) state via the
// React master toggle must start the live watchdog immediately, without a
// restart. The toggle posts watchdog_idle_enabled/busy_enabled=true while
// the vestigial watchdog_enabled stays false (it was loaded false). The
// old handler keyed its stop decision off that raw watchdog_enabled=false
// and called StopWatchdog(), so the watchdog never started until restart.
It("starts the live watchdog on a cold enable even when watchdog_enabled=false", func() {
Expect(app.ModelLoader().GetWatchDog()).To(BeNil(), "precondition: watchdog should be off")

rec := post(`{"watchdog_enabled":false,"watchdog_idle_enabled":true,"watchdog_busy_enabled":true,"watchdog_idle_timeout":"15m","watchdog_busy_timeout":"5m","watchdog_interval":"1s"}`)
Expect(rec.Code).To(Equal(http.StatusOK))

Expect(app.ModelLoader().GetWatchDog()).ToNot(BeNil(),
"watchdog should be running after a cold enable, without waiting for a restart")
})
})
2 changes: 1 addition & 1 deletion core/http/react-ui/src/pages/Settings.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ export default function Settings() {
</h3>
<div className="card">
<SettingRow label="Enable Watchdog" description="Automatically monitor and manage backend processes">
<Toggle checked={settings.watchdog_idle_enabled || settings.watchdog_busy_enabled} onChange={(v) => { update('watchdog_idle_enabled', v); update('watchdog_busy_enabled', v) }} />
<Toggle checked={settings.watchdog_idle_enabled || settings.watchdog_busy_enabled} onChange={(v) => { update('watchdog_idle_enabled', v); update('watchdog_busy_enabled', v); update('watchdog_enabled', v) }} />
</SettingRow>
<SettingRow label="Enable Idle Check" description="Automatically stop backends that have been idle too long">
<Toggle checked={settings.watchdog_idle_enabled} onChange={(v) => update('watchdog_idle_enabled', v)} disabled={!watchdogEnabled} />
Expand Down
Loading