feat: backend versioning, upgrade detection and auto-upgrade

core/application/application.go

@@ -37,6 +37,9 @@ type Application struct {
 
 	// Distributed mode services (nil when not in distributed mode)
 	distributed *DistributedServices
+
+	// Upgrade checker (background service for detecting backend upgrades)
+	upgradeChecker *UpgradeChecker
 }
 
 func newApplication(appConfig *config.ApplicationConfig) *Application {
@@ -79,6 +82,19 @@ func (a *Application) AgentJobService() *agentpool.AgentJobService {
 	return a.agentJobService
 }
 
+func (a *Application) UpgradeChecker() *UpgradeChecker {
+	return a.upgradeChecker
+}
+
+// distributedDB returns the PostgreSQL database for distributed coordination,
+// or nil in standalone mode.
+func (a *Application) distributedDB() *gorm.DB {
+	if a.distributed != nil {
+		return a.authDB
+	}
+	return nil
+}
+
 func (a *Application) AgentPoolService() *agentpool.AgentPoolService {
 	return a.agentPoolService.Load()
 }

core/application/config_file_watcher.go

@@ -335,6 +335,9 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
 			if settings.AutoloadBackendGalleries != nil && !envAutoloadBackendGalleries {
 				appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries
 			}
+			if settings.AutoUpgradeBackends != nil {
+				appConfig.AutoUpgradeBackends = *settings.AutoUpgradeBackends
+			}
 			if settings.ApiKeys != nil {
 				// API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys
 				// If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys

core/application/startup.go

@@ -231,6 +231,15 @@ func New(opts ...config.AppOption) (*Application, error) {
 		xlog.Error("error registering external backends", "error", err)
 	}
 
+	// Start background upgrade checker for backends.
+	// In distributed mode, uses PostgreSQL advisory lock so only one frontend
+	// instance runs periodic checks (avoids duplicate upgrades across replicas).
+	if len(options.BackendGalleries) > 0 {
+		uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB())
+		application.upgradeChecker = uc
+		go uc.Run(options.Context)
+	}
+
 	if options.ConfigFile != "" {
 		if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
 			xlog.Error("error loading config file", "error", err)

core/application/upgrade_checker.go

@@ -0,0 +1,198 @@
+package application
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/services/advisorylock"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/mudler/xlog"
+	"gorm.io/gorm"
+)
+
+// UpgradeChecker periodically checks for backend upgrades and optionally
+// auto-upgrades them. It caches the last check results for API queries.
+//
+// In standalone mode it runs a simple ticker loop.
+// In distributed mode it uses a PostgreSQL advisory lock so that only one
+// frontend instance performs periodic checks and auto-upgrades at a time.
+type UpgradeChecker struct {
+	appConfig   *config.ApplicationConfig
+	modelLoader *model.ModelLoader
+	galleries   []config.Gallery
+	systemState *system.SystemState
+	db          *gorm.DB // non-nil in distributed mode
+
+	checkInterval time.Duration
+	stop          chan struct{}
+	done          chan struct{}
+	triggerCh     chan struct{}
+
+	mu            sync.RWMutex
+	lastUpgrades  map[string]gallery.UpgradeInfo
+	lastCheckTime time.Time
+}
+
+// NewUpgradeChecker creates a new UpgradeChecker service.
+// Pass db=nil for standalone mode, or a *gorm.DB for distributed mode
+// (uses advisory locks so only one instance runs periodic checks).
+func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB) *UpgradeChecker {
+	return &UpgradeChecker{
+		appConfig:     appConfig,
+		modelLoader:   ml,
+		galleries:     appConfig.BackendGalleries,
+		systemState:   appConfig.SystemState,
+		db:            db,
+		checkInterval: 6 * time.Hour,
+		stop:          make(chan struct{}),
+		done:          make(chan struct{}),
+		triggerCh:     make(chan struct{}, 1),
+		lastUpgrades:  make(map[string]gallery.UpgradeInfo),
+	}
+}
+
+// Run starts the upgrade checker loop. It waits 30 seconds after startup,
+// performs an initial check, then re-checks every 6 hours.
+//
+// In distributed mode, periodic checks are guarded by a PostgreSQL advisory
+// lock so only one frontend instance runs them. On-demand triggers (TriggerCheck)
+// and the initial check always run locally for fast API response cache warming.
+func (uc *UpgradeChecker) Run(ctx context.Context) {
+	defer close(uc.done)
+
+	// Initial delay: don't slow down startup
+	select {
+	case <-ctx.Done():
+		return
+	case <-uc.stop:
+		return
+	case <-time.After(30 * time.Second):
+	}
+
+	// First check always runs locally (to warm the cache on this instance)
+	uc.runCheck(ctx)
+
+	if uc.db != nil {
+		// Distributed mode: use advisory lock for periodic checks.
+		// RunLeaderLoop ticks every checkInterval; only the lock holder executes.
+		go advisorylock.RunLeaderLoop(ctx, uc.db, advisorylock.KeyBackendUpgradeCheck, uc.checkInterval, func() {
+			uc.runCheck(ctx)
+		})
+
+		// Still listen for on-demand triggers (from API / settings change)
+		// and stop signal — these run on every instance.
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-uc.stop:
+				return
+			case <-uc.triggerCh:
+				uc.runCheck(ctx)
+			}
+		}
+	} else {
+		// Standalone mode: simple ticker loop
+		ticker := time.NewTicker(uc.checkInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-uc.stop:
+				return
+			case <-ticker.C:
+				uc.runCheck(ctx)
+			case <-uc.triggerCh:
+				uc.runCheck(ctx)
+			}
+		}
+	}
+}
+
+// Shutdown stops the upgrade checker loop.
+func (uc *UpgradeChecker) Shutdown() {
+	close(uc.stop)
+	<-uc.done
+}
+
+// TriggerCheck forces an immediate upgrade check on this instance.
+func (uc *UpgradeChecker) TriggerCheck() {
+	select {
+	case uc.triggerCh <- struct{}{}:
+	default:
+		// Already triggered, skip
+	}
+}
+
+// GetAvailableUpgrades returns the cached upgrade check results.
+func (uc *UpgradeChecker) GetAvailableUpgrades() map[string]gallery.UpgradeInfo {
+	uc.mu.RLock()
+	defer uc.mu.RUnlock()
+
+	// Return a copy to avoid races
+	result := make(map[string]gallery.UpgradeInfo, len(uc.lastUpgrades))
+	for k, v := range uc.lastUpgrades {
+		result[k] = v
+	}
+	return result
+}
+
+func (uc *UpgradeChecker) runCheck(ctx context.Context) {
+	upgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState)
+
+	uc.mu.Lock()
+	uc.lastCheckTime = time.Now()
+	if err != nil {
+		xlog.Debug("Backend upgrade check failed", "error", err)
+		uc.mu.Unlock()
+		return
+	}
+	uc.lastUpgrades = upgrades
+	uc.mu.Unlock()
+
+	if len(upgrades) == 0 {
+		xlog.Debug("All backends up to date")
+		return
+	}
+
+	// Log available upgrades
+	for name, info := range upgrades {
+		if info.AvailableVersion != "" {
+			xlog.Info("Backend upgrade available",
+				"backend", name,
+				"installed", info.InstalledVersion,
+				"available", info.AvailableVersion)
+		} else {
+			xlog.Info("Backend upgrade available (new build)",
+				"backend", name)
+		}
+	}
+
+	// Auto-upgrade if enabled
+	if uc.appConfig.AutoUpgradeBackends {
+		for name, info := range upgrades {
+			xlog.Info("Auto-upgrading backend", "backend", name,
+				"from", info.InstalledVersion, "to", info.AvailableVersion)
+			if err := gallery.UpgradeBackend(ctx, uc.systemState, uc.modelLoader,
+				uc.galleries, name, nil); err != nil {
+				xlog.Error("Failed to auto-upgrade backend",
+					"backend", name, "error", err)
+			} else {
+				xlog.Info("Backend upgraded successfully", "backend", name,
+					"version", info.AvailableVersion)
+			}
+		}
+		// Re-check to update cache after upgrades
+		if freshUpgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState); err == nil {
+			uc.mu.Lock()
+			uc.lastUpgrades = freshUpgrades
+			uc.mu.Unlock()
+		}
+	}
+}

core/cli/backends.go

@@ -40,10 +40,17 @@ type BackendsUninstall struct {
 	BackendsCMDFlags `embed:""`
 }
 
+type BackendsUpgrade struct {
+	BackendArgs []string `arg:"" optional:"" name:"backends" help:"Backend names to upgrade (empty = upgrade all)"`
+
+	BackendsCMDFlags `embed:""`
+}
+
 type BackendsCMD struct {
 	List      BackendsList      `cmd:"" help:"List the backends available in your galleries" default:"withargs"`
 	Install   BackendsInstall   `cmd:"" help:"Install a backend from the gallery"`
 	Uninstall BackendsUninstall `cmd:"" help:"Uninstall a backend"`
+	Upgrade   BackendsUpgrade   `cmd:"" help:"Upgrade backends to latest versions"`
 }
 
 func (bl *BackendsList) Run(ctx *cliContext.Context) error {
@@ -64,11 +71,27 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error {
 	if err != nil {
 		return err
 	}
+
+	// Check for upgrades
+	upgrades, _ := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState)
+
 	for _, backend := range backends {
+		versionStr := ""
+		if backend.Version != "" {
+			versionStr = " v" + backend.Version
+		}
 		if backend.Installed {
-			fmt.Printf(" * %s@%s (installed)\n", backend.Gallery.Name, backend.Name)
+			if info, ok := upgrades[backend.Name]; ok {
+				upgradeStr := info.AvailableVersion
+				if upgradeStr == "" {
+					upgradeStr = "new build"
+				}
+				fmt.Printf(" * %s@%s%s (installed, upgrade available: %s)\n", backend.Gallery.Name, backend.Name, versionStr, upgradeStr)
+			} else {
+				fmt.Printf(" * %s@%s%s (installed)\n", backend.Gallery.Name, backend.Name, versionStr)
+			}
 		} else {
-			fmt.Printf(" - %s@%s\n", backend.Gallery.Name, backend.Name)
+			fmt.Printf(" - %s@%s%s\n", backend.Gallery.Name, backend.Name, versionStr)
 		}
 	}
 	return nil
@@ -111,6 +134,79 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
 	return nil
 }
 
+func (bu *BackendsUpgrade) Run(ctx *cliContext.Context) error {
+	var galleries []config.Gallery
+	if err := json.Unmarshal([]byte(bu.BackendGalleries), &galleries); err != nil {
+		xlog.Error("unable to load galleries", "error", err)
+	}
+
+	systemState, err := system.GetSystemState(
+		system.WithBackendSystemPath(bu.BackendsSystemPath),
+		system.WithBackendPath(bu.BackendsPath),
+	)
+	if err != nil {
+		return err
+	}
+
+	upgrades, err := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState)
+	if err != nil {
+		return fmt.Errorf("failed to check for upgrades: %w", err)
+	}
+
+	if len(upgrades) == 0 {
+		fmt.Println("All backends are up to date.")
+		return nil
+	}
+
+	// Filter to specified backends if args given
+	toUpgrade := upgrades
+	if len(bu.BackendArgs) > 0 {
+		toUpgrade = make(map[string]gallery.UpgradeInfo)
+		for _, name := range bu.BackendArgs {
+			if info, ok := upgrades[name]; ok {
+				toUpgrade[name] = info
+			} else {
+				fmt.Printf("Backend %s: no upgrade available\n", name)
+			}
+		}
+	}
+
+	if len(toUpgrade) == 0 {
+		fmt.Println("No upgrades to apply.")
+		return nil
+	}
+
+	modelLoader := model.NewModelLoader(systemState)
+	for name, info := range toUpgrade {
+		versionStr := ""
+		if info.AvailableVersion != "" {
+			versionStr = " to v" + info.AvailableVersion
+		}
+		fmt.Printf("Upgrading %s%s...\n", name, versionStr)
+
+		progressBar := progressbar.NewOptions(
+			1000,
+			progressbar.OptionSetDescription(fmt.Sprintf("downloading %s", name)),
+			progressbar.OptionShowBytes(false),
+			progressbar.OptionClearOnFinish(),
+		)
+		progressCallback := func(fileName string, current string, total string, percentage float64) {
+			v := int(percentage * 10)
+			if err := progressBar.Set(v); err != nil {
+				xlog.Error("error updating progress bar", "error", err)
+			}
+		}
+
+		if err := gallery.UpgradeBackend(context.Background(), systemState, modelLoader, galleries, name, progressCallback); err != nil {
+			fmt.Printf("Failed to upgrade %s: %v\n", name, err)
+		} else {
+			fmt.Printf("Backend %s upgraded successfully\n", name)
+		}
+	}
+
+	return nil
+}
+
 func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
 	for _, backendName := range bu.BackendArgs {
 		xlog.Info("uninstalling backend", "backend", backendName)