do not flood of healthy checks

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

pkg/model/loader.go

@@ -326,6 +326,15 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model {
 	}
 
 	xlog.Debug("Model already loaded in memory", "model", s)
+
+	// Skip the gRPC health check if the model was recently verified.
+	// This avoids serializing concurrent requests behind ml.mu while each
+	// one does a network round-trip (especially costly in distributed mode).
+	if m.IsRecentlyHealthy() {
+		xlog.Debug("Model health check cached, skipping gRPC probe", "model", s)
+		return m
+	}
+
 	client := m.GRPC(false, ml.wd)
 
 	xlog.Debug("Checking model availability", "model", s)
@@ -352,5 +361,6 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model {
 		}
 	}
 
+	m.MarkHealthy()
 	return m
 }

pkg/model/model.go

@@ -2,16 +2,23 @@ package model
 
 import (
 	"sync"
+	"time"
 
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
 	process "github.com/mudler/go-processmanager"
 )
 
+// healthCheckTTL is the duration for which a successful health check is cached.
+// Subsequent checkIsLoaded calls within this window skip the gRPC round-trip,
+// avoiding serialization of concurrent requests behind ml.mu.Lock().
+const healthCheckTTL = 30 * time.Second
+
 type Model struct {
-	ID      string `json:"id"`
-	address string
-	client  grpc.Backend
-	process *process.Process
+	ID              string `json:"id"`
+	address         string
+	client          grpc.Backend
+	process         *process.Process
+	lastHealthCheck time.Time
 	sync.Mutex
 }
 
@@ -37,6 +44,20 @@ func (m *Model) Process() *process.Process {
 	return m.process
 }
 
+// IsRecentlyHealthy returns true if the model passed a health check within the TTL.
+func (m *Model) IsRecentlyHealthy() bool {
+	m.Lock()
+	defer m.Unlock()
+	return !m.lastHealthCheck.IsZero() && time.Since(m.lastHealthCheck) < healthCheckTTL
+}
+
+// MarkHealthy records the current time as the last successful health check.
+func (m *Model) MarkHealthy() {
+	m.Lock()
+	defer m.Unlock()
+	m.lastHealthCheck = time.Now()
+}
+
 func (m *Model) GRPC(parallel bool, wd *WatchDog) grpc.Backend {
 	if m.client != nil {
 		return m.client