mudler
diff --git a/‎core/application/distributed.go‎
Lines changed: 4 additions & 3 deletions b/‎core/application/distributed.go‎
Lines changed: 4 additions & 3 deletions
diff --git a/‎core/application/startup.go‎
Lines changed: 8 additions & 0 deletions b/‎core/application/startup.go‎
Lines changed: 8 additions & 0 deletions
diff --git a/‎core/services/messaging/subjects.go‎
Lines changed: 30 additions & 0 deletions b/‎core/services/messaging/subjects.go‎
Lines changed: 30 additions & 0 deletions
diff --git a/‎core/services/nodes/local_stub_invalidator.go‎
Lines changed: 58 additions & 0 deletions b/‎core/services/nodes/local_stub_invalidator.go‎
Lines changed: 58 additions & 0 deletions
diff --git a/‎core/services/nodes/local_stub_invalidator_test.go‎
Lines changed: 109 additions & 0 deletions b/‎core/services/nodes/local_stub_invalidator_test.go‎
Lines changed: 109 additions & 0 deletions
diff --git a/‎core/services/nodes/model_router_test.go‎
Lines changed: 10 additions & 7 deletions b/‎core/services/nodes/model_router_test.go‎
Lines changed: 10 additions & 7 deletions
@@ -286,13 +286,14 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
 		prefixProvider = prefixSync
 
 		// Invalidate the prefix-cache index whenever a replica row is removed.
-		// SetReplicaRemovedHook fires from the single chokepoint all removal paths
+		// AddReplicaRemovedHook fires from the single chokepoint all removal paths
 		// funnel through (RemoveNodeModel / RemoveAllNodeModelReplicas), so this
 		// one hook covers every path: reconciler scale-down, probe reaper,
 		// health-monitor reap, RemoteUnloaderAdapter, and the router. Registering
 		// it only inside this enabled block keeps the disabled path a true no-op
-		// (the registry stays hook-less).
-		registry.SetReplicaRemovedHook(func(model, node string, replica int) {
+		// for the prefix cache; other subsystems register their own hooks
+		// independently and are unaffected either way.
+		registry.AddReplicaRemovedHook(func(model, node string, replica int) {
 			if replica < 0 {
 				prefixSync.InvalidateNode(model, node)
 			} else {
 
@@ -283,6 +283,14 @@ func New(opts ...config.AppOption) (*Application, error) {
 			distSvc.Registry,
 		)
 		application.modelLoader.SetModelStore(distStore)
+		// Drop the local stub when a model's last replica leaves the registry.
+		// The store reports local stubs UNION registry rows, and every removal
+		// path deletes the row only, so without this the frontend keeps
+		// reporting a model as loaded long after the replica is gone.
+		// Registered unconditionally: this is independent of the prefix cache.
+		distSvc.Registry.AddReplicaRemovedHook(
+			nodes.NewLocalStubInvalidator(distSvc.Registry, distStore),
+		)
 		// Start health monitor
 		distSvc.Health.Start(options.Context)
 		// Start replica reconciler for auto-scaling model replicas
 
@@ -365,6 +365,36 @@ type ModelDeleteReply struct {
 	Error   string `json:"error,omitempty"`
 }
 
+// SubjectNodeModelsRunning asks a worker node which model backend processes it
+// currently has running. Uses NATS request-reply.
+//
+// This is the authoritative answer to "is this replica still alive". The worker
+// owns the process table, so unlike a health probe against the backend's own
+// serving port, its reply does not depend on whether that backend happens to be
+// busy: a model mid-generation cannot answer a gRPC health check for minutes at
+// a time, but the worker answers immediately either way.
+func SubjectNodeModelsRunning(nodeID string) string {
+	return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".models.running"
+}
+
+// ModelsRunningRequest is the payload for a models.running NATS request.
+type ModelsRunningRequest struct{}
+
+// ModelsRunningReply is the response from a models.running NATS request.
+type ModelsRunningReply struct {
+	Models []RunningModelInfo `json:"models"`
+	Error  string             `json:"error,omitempty"`
+}
+
+// RunningModelInfo identifies one live backend process on a worker. The triple
+// is isomorphic to a controller NodeModel row's (model_name, replica_index,
+// address), which is what lets the reconciler diff the two directly.
+type RunningModelInfo struct {
+	ModelID      string `json:"model_id"`
+	ReplicaIndex int    `json:"replica_index"`
+	Address      string `json:"address,omitempty"`
+}
+
 // SubjectNodeStop tells a serve-backend node to shut down entirely
 // (deregister + exit). The node will not restart the backend process.
 func SubjectNodeStop(nodeID string) string {
 
@@ -0,0 +1,58 @@
+package nodes
+
+import (
+	"context"
+
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/xlog"
+)
+
+// NewLocalStubInvalidator returns a replica-removed hook that drops the
+// frontend's in-process stub for a model once no healthy replica of it remains
+// anywhere in the cluster.
+//
+// Why this is needed: in distributed mode every routed model leaves a stub in
+// the frontend's ModelLoader, and DistributedModelStore.Range reports local
+// stubs UNION the registry rows. Every registry removal path deletes only the
+// DB row, so without this hook the stub outlives the replica and the model is
+// reported as loaded forever. That is the "loaded on the home page, absent from
+// every node" ghost: /system reads the union and still sees the stub, while
+// /api/nodes/models reads the registry and correctly sees nothing.
+//
+// The stub is dropped only when the LAST replica is gone. While another node
+// still serves the model the frontend is right to consider it loaded, and each
+// request re-routes through SmartRouter to pick a live replica anyway.
+//
+// This deletes the store entry directly rather than going through
+// ShutdownModel: the replica is already gone from the registry, so there is
+// nothing left to unload, and ShutdownModel would emit a pointless remote
+// unload to a node that no longer hosts it.
+func NewLocalStubInvalidator(registry *NodeRegistry, store model.ModelStore) func(modelID, nodeID string, replicaIndex int) {
+	return func(modelID, nodeID string, replicaIndex int) {
+		if registry == nil || store == nil || modelID == "" {
+			return
+		}
+		if _, ok := store.Get(modelID); !ok {
+			return // no local stub for this model, nothing to invalidate
+		}
+
+		// Registry rows are the source of truth for "is this model loaded
+		// anywhere". Only healthy nodes with a loaded row are returned.
+		remaining, err := registry.FindNodesWithModel(context.Background(), modelID)
+		if err != nil {
+			// Leave the stub alone rather than risk dropping a live model on a
+			// transient DB error. A later removal re-runs this check, and the
+			// reconciler keeps probing, so the ghost is not permanent.
+			xlog.Warn("Local stub invalidation skipped: failed to count remaining replicas",
+				"model", modelID, "node", nodeID, "replica", replicaIndex, "error", err)
+			return
+		}
+		if len(remaining) > 0 {
+			return
+		}
+
+		store.Delete(modelID)
+		xlog.Info("Dropped local model stub after its last replica was removed",
+			"model", modelID, "node", nodeID, "replica", replicaIndex)
+	}
+}
@@ -0,0 +1,109 @@
+package nodes
+
+import (
+	"context"
+	"runtime"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gorm.io/gorm"
+
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/core/services/testutil"
+)
+
+// In distributed mode the frontend keeps an in-process stub for every model it
+// has routed, and DistributedModelStore.Range reports local stubs UNION the
+// registry rows. Every registry removal path deletes the DB row only, so
+// without an invalidator the stub outlives the replica and the model is
+// reported as loaded forever (visible on the home page, absent from the nodes
+// page). These tests pin the invalidator that closes that gap.
+var _ = Describe("LocalStubInvalidator", func() {
+	var (
+		db       *gorm.DB
+		registry *NodeRegistry
+		local    *model.InMemoryModelStore
+		nodeA    *BackendNode
+		nodeB    *BackendNode
+	)
+
+	BeforeEach(func() {
+		if runtime.GOOS == "darwin" {
+			Skip("testcontainers requires Docker, not available on macOS CI")
+		}
+		db = testutil.SetupTestDB()
+		var err error
+		registry, err = NewNodeRegistry(db)
+		Expect(err).ToNot(HaveOccurred())
+		local = model.NewInMemoryModelStore()
+		nodeA = &BackendNode{Name: "node-a", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"}
+		nodeB = &BackendNode{Name: "node-b", NodeType: NodeTypeBackend, Address: "10.0.0.2:50051"}
+		Expect(registry.Register(context.Background(), nodeA, true)).To(Succeed())
+		Expect(registry.Register(context.Background(), nodeB, true)).To(Succeed())
+	})
+
+	It("drops the local stub once the last replica of the model is gone", func() {
+		store := NewDistributedModelStore(local, registry)
+		Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "ghost-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed())
+		local.Set("ghost-model", model.NewModel("ghost-model", "10.0.0.1:12345", nil))
+
+		registry.AddReplicaRemovedHook(NewLocalStubInvalidator(registry, store))
+		Expect(registry.RemoveNodeModel(context.Background(), nodeA.ID, "ghost-model", 0)).To(Succeed())
+
+		_, ok := local.Get("ghost-model")
+		Expect(ok).To(BeFalse(), "the stub must not outlive the last replica of the model")
+
+		// And the model must disappear from the loaded listing that feeds /system.
+		listed := []string{}
+		store.Range(func(id string, _ *model.Model) bool {
+			listed = append(listed, id)
+			return true
+		})
+		Expect(listed).ToNot(ContainElement("ghost-model"))
+	})
+
+	It("keeps the local stub while another replica still serves the model", func() {
+		store := NewDistributedModelStore(local, registry)
+		Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "shared-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed())
+		Expect(registry.SetNodeModel(context.Background(), nodeB.ID, "shared-model", 0, "loaded", "10.0.0.2:12345", 0)).To(Succeed())
+		local.Set("shared-model", model.NewModel("shared-model", "10.0.0.1:12345", nil))
+
+		registry.AddReplicaRemovedHook(NewLocalStubInvalidator(registry, store))
+		Expect(registry.RemoveNodeModel(context.Background(), nodeA.ID, "shared-model", 0)).To(Succeed())
+
+		_, ok := local.Get("shared-model")
+		Expect(ok).To(BeTrue(), "a model still loaded on another node must stay in the local store")
+	})
+
+	It("fires every registered replica-removed hook", func() {
+		Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "m", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed())
+
+		var first, second int
+		registry.AddReplicaRemovedHook(func(_, _ string, _ int) { first++ })
+		registry.AddReplicaRemovedHook(func(_, _ string, _ int) { second++ })
+
+		Expect(registry.RemoveNodeModel(context.Background(), nodeA.ID, "m", 0)).To(Succeed())
+
+		// Prefix-cache invalidation and local-stub invalidation are independent
+		// subsystems; registering one must never silently displace the other.
+		Expect(first).To(Equal(1))
+		Expect(second).To(Equal(1))
+	})
+
+	It("drops the local stub when a whole node's replicas are removed", func() {
+		store := NewDistributedModelStore(local, registry)
+		Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "node-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed())
+		local.Set("node-model", model.NewModel("node-model", "10.0.0.1:12345", nil))
+
+		registry.AddReplicaRemovedHook(NewLocalStubInvalidator(registry, store))
+		// Negative replica index signals "all replicas of this model on the node",
+		// the shape used by node deregistration and health reaping.
+		Expect(registry.RemoveAllNodeModelReplicas(context.Background(), nodeA.ID, "node-model")).To(Succeed())
+
+		Eventually(func() bool {
+			_, ok := local.Get("node-model")
+			return ok
+		}, time.Second, 10*time.Millisecond).Should(BeFalse())
+	})
+})
@@ -197,19 +197,22 @@ var _ = Describe("ModelRouterAdapter", func() {
 			adapter.mu.Unlock()
 			Expect(hasRelease).To(BeTrue())
 
-			// The initial in-flight reservation is released via OnFirstComplete after
-			// the first inference call, not during ReleaseModel. ReleaseModel only
-			// closes the client.
+			// The initial in-flight reservation is released by whichever comes
+			// first: the first inference completing, or the route being released.
+			// Nothing has happened yet, so it is still held.
 			fakeReg.mu.Lock()
 			countBeforeRelease := fakeReg.decrementCalled["node-1:test-model"]
 			fakeReg.mu.Unlock()
 			Expect(countBeforeRelease).To(Equal(0))
 
+			// Releasing the route without any inference must give it back, or the
+			// counter leaks and pins the replica against every eviction query.
 			adapter.ReleaseModel("test-model")
-			fakeReg.mu.Lock()
-			countAfterRelease := fakeReg.decrementCalled["node-1:test-model"]
-			fakeReg.mu.Unlock()
-			Expect(countAfterRelease).To(Equal(0))
+			Eventually(func() int {
+				fakeReg.mu.Lock()
+				defer fakeReg.mu.Unlock()
+				return fakeReg.decrementCalled["node-1:test-model"]
+			}).Should(Equal(1))
 		})
 	})
 })