diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index cb45072494e5..5afd77096b0d 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -349,12 +349,52 @@ func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityP return capabilities } +// ClusterInstalledProvider reports the backends installed somewhere in the +// cluster. It is nil in single-node deployments, where the local filesystem is +// the only install state that exists. +type ClusterInstalledProvider func(ctx context.Context) ([]string, error) + +// resolveClusterInstalled reads the backends installed across the cluster, +// degrading to the local-only view on error. +// +// Every discovery endpoint that filters on installed-state shares this: a +// backend lives on the worker node that runs it, so the controller's own +// filesystem reports it missing and the endpoint hides it. A nil set leaves +// the local filesystem as the only source, so single-node listings are +// untouched, and a registry hiccup degrades to that same listing rather than +// erroring the request. +func resolveClusterInstalled(ctx context.Context, provider ClusterInstalledProvider) map[string]struct{} { + if provider == nil { + return nil + } + names, err := provider(ctx) + if err != nil { + xlog.Warn("Could not read cluster backend install state, reporting the local system only", "error", err) + return nil + } + installed := make(map[string]struct{}, len(names)) + for _, name := range names { + installed[name] = struct{}{} + } + return installed +} + +// installedInCluster reports whether a backend is installed on the host serving +// the listing or on any node of the cluster. +func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[string]struct{}) bool { + if backend.Installed { + return true + } + _, ok := clusterInstalled[backend.Name] + return ok +} + // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI // @Summary List all available Backends // @Tags backends // @Success 200 {object} []gallery.GalleryBackend "Response" // @Router /backends/available [get] -func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc { +func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) @@ -362,6 +402,12 @@ func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *sy if err != nil { return err } + + installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled) + for _, b := range backends { + b.SetInstalled(installedInCluster(b, installed)) + } + return c.JSON(200, backends) } } diff --git a/core/http/endpoints/localai/backend_available_cluster_test.go b/core/http/endpoints/localai/backend_available_cluster_test.go index 26b2247f9fc9..26e1d96b87c0 100644 --- a/core/http/endpoints/localai/backend_available_cluster_test.go +++ b/core/http/endpoints/localai/backend_available_cluster_test.go @@ -66,7 +66,7 @@ var _ = Describe("ListAvailableBackendsEndpoint cluster capabilities", func() { listNames := func(provider ClusterCapabilityProvider) []string { svc := CreateBackendEndpointService(galleries, systemState, nil, nil) - app.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, provider)) + app.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, provider, nil)) req := httptest.NewRequest(http.MethodGet, "/backends/available", nil) rec := httptest.NewRecorder() diff --git a/core/http/endpoints/localai/discovery_cluster_capabilities_test.go b/core/http/endpoints/localai/discovery_cluster_capabilities_test.go index fb4e5928ea2e..f414038c29d2 100644 --- a/core/http/endpoints/localai/discovery_cluster_capabilities_test.go +++ b/core/http/endpoints/localai/discovery_cluster_capabilities_test.go @@ -100,26 +100,26 @@ var _ = Describe("Tagged backend discovery with cluster capabilities", func() { Describe("fine-tune backends", func() { It("hides GPU-only backends with no cluster provider", func() { - names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil)) + names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil, nil)) Expect(names).To(ContainElement("cpu-trainer")) Expect(names).NotTo(ContainElement("gpu-trainer")) }) It("lists GPU-only backends a worker node can run", func() { - names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker)) + names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, nil)) Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer")) }) }) Describe("quantization backends", func() { It("hides GPU-only backends with no cluster provider", func() { - names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil)) + names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil, nil)) Expect(names).To(ContainElement("cpu-trainer")) Expect(names).NotTo(ContainElement("gpu-trainer")) }) It("lists GPU-only backends a worker node can run", func() { - names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker)) + names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker, nil)) Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer")) }) }) diff --git a/core/http/endpoints/localai/discovery_cluster_installed_test.go b/core/http/endpoints/localai/discovery_cluster_installed_test.go new file mode 100644 index 000000000000..2dd44e8801e3 --- /dev/null +++ b/core/http/endpoints/localai/discovery_cluster_installed_test.go @@ -0,0 +1,174 @@ +package localai_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + . "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +// Installed-state on a distributed controller is derived from the controller's +// own filesystem, where a backend installed on a GPU worker does not exist. +// The capability fix (#10947) made those backends *listable*; every surface +// that filters on installed-state still dropped them, so an admin with a +// fine-tuning-capable GPU worker got an empty dropdown. +var _ = Describe("Backend discovery with cluster install state", func() { + var ( + appCfg *config.ApplicationConfig + systemState *system.SystemState + galleries []config.Gallery + tmpDir string + ) + + BeforeEach(func() { + var err error + tmpDir, err = os.MkdirTemp("", "discovery-cluster-installed-*") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + Expect(os.RemoveAll(tmpDir)).To(Succeed()) + }) + + // cpu-trainer sits on the controller's disk; gpu-trainer exists only on + // a worker, so nothing on this filesystem can prove it is installed. + writeFakeSystemBackend(tmpDir, "cpu-trainer") + + galleryPath := filepath.Join(tmpDir, "gallery.yaml") + data, err := yaml.Marshal([]gallery.GalleryBackend{ + { + Metadata: gallery.Metadata{ + Name: "gpu-trainer", + Tags: []string{"fine-tuning", "quantization"}, + }, + CapabilitiesMap: map[string]string{"nvidia-cuda-13": "gpu-trainer-cuda-13"}, + }, + { + Metadata: gallery.Metadata{ + Name: "cpu-trainer", + Tags: []string{"fine-tuning", "quantization"}, + }, + CapabilitiesMap: map[string]string{"default": "cpu-trainer-cpu"}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(galleryPath, data, 0o644)).To(Succeed()) + + galleries = []config.Gallery{{Name: "test-gallery", URL: "file://" + galleryPath}} + // A GPU-less controller pod: capability "default". + systemState = system.NewCapabilityState("default", + system.WithBackendPath(tmpDir), system.WithBackendSystemPath(tmpDir)) + appCfg = &config.ApplicationConfig{ + BackendGalleries: galleries, + SystemState: systemState, + } + }) + + nvidiaWorker := func(ctx context.Context) ([]string, error) { + return []string{"nvidia-cuda-13"}, nil + } + installedOnWorker := func(ctx context.Context) ([]string, error) { + return []string{"gpu-trainer"}, nil + } + registryDown := func(ctx context.Context) ([]string, error) { + return nil, errors.New("registry unavailable") + } + + // listNames drives handler over its route and returns the backend names. + listNames := func(path string, handler echo.HandlerFunc) []string { + e := echo.New() + e.GET(path, handler) + + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var backends []struct { + Name string `json:"name"` + } + Expect(json.Unmarshal(rec.Body.Bytes(), &backends)).To(Succeed()) + + names := []string{} + for _, b := range backends { + names = append(names, b.Name) + } + return names + } + + Describe("fine-tune backends", func() { + It("lists a backend installed only on a worker node", func() { + names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, installedOnWorker)) + Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer")) + }) + + It("keeps single-node listings unchanged with no provider", func() { + names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil, nil)) + Expect(names).To(ContainElement("cpu-trainer")) + Expect(names).NotTo(ContainElement("gpu-trainer")) + }) + + It("degrades to the local listing when the registry errors", func() { + names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, registryDown)) + Expect(names).To(ContainElement("cpu-trainer")) + Expect(names).NotTo(ContainElement("gpu-trainer")) + }) + }) + + Describe("quantization backends", func() { + It("lists a backend installed only on a worker node", func() { + names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker, installedOnWorker)) + Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer")) + }) + + It("keeps single-node listings unchanged with no provider", func() { + names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil, nil)) + Expect(names).To(ContainElement("cpu-trainer")) + Expect(names).NotTo(ContainElement("gpu-trainer")) + }) + }) + + Describe("GET /backends/available", func() { + installedFlags := func(capabilities ClusterCapabilityProvider, installed ClusterInstalledProvider) map[string]bool { + e := echo.New() + svc := CreateBackendEndpointService(galleries, systemState, nil, nil) + e.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, capabilities, installed)) + + req := httptest.NewRequest(http.MethodGet, "/backends/available", nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var backends []gallery.GalleryBackend + Expect(json.Unmarshal(rec.Body.Bytes(), &backends)).To(Succeed()) + + flags := map[string]bool{} + for _, b := range backends { + flags[b.Name] = b.Installed + } + return flags + } + + It("reports a worker-installed backend as installed", func() { + flags := installedFlags(nvidiaWorker, installedOnWorker) + Expect(flags).To(HaveKeyWithValue("gpu-trainer", true)) + Expect(flags).To(HaveKeyWithValue("cpu-trainer", true)) + }) + + It("keeps single-node install state unchanged with no provider", func() { + flags := installedFlags(nvidiaWorker, nil) + Expect(flags).To(HaveKeyWithValue("gpu-trainer", false)) + Expect(flags).To(HaveKeyWithValue("cpu-trainer", true)) + }) + }) +}) diff --git a/core/http/endpoints/localai/finetune.go b/core/http/endpoints/localai/finetune.go index 01cb413e531c..4948b65fcecb 100644 --- a/core/http/endpoints/localai/finetune.go +++ b/core/http/endpoints/localai/finetune.go @@ -274,9 +274,10 @@ func DownloadExportedModelEndpoint(ftService *finetune.FineTuneService) echo.Han } // ListFineTuneBackendsEndpoint returns installed backends tagged with "fine-tuning". -func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc { +func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) + installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled) backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities) if err != nil { return c.JSON(http.StatusInternalServerError, map[string]string{ @@ -292,7 +293,7 @@ func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCa var result []backendInfo for _, b := range backends { - if !b.Installed { + if !installedInCluster(b, installed) { continue } hasTag := false diff --git a/core/http/endpoints/localai/quantization.go b/core/http/endpoints/localai/quantization.go index ca6aced15d69..175bf974081d 100644 --- a/core/http/endpoints/localai/quantization.go +++ b/core/http/endpoints/localai/quantization.go @@ -193,9 +193,10 @@ func DownloadQuantizedModelEndpoint(qService *quantization.QuantizationService) } // ListQuantizationBackendsEndpoint returns installed backends tagged with "quantization". -func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc { +func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) + installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled) backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities) if err != nil { return c.JSON(http.StatusInternalServerError, map[string]string{ @@ -211,7 +212,7 @@ func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clust var result []backendInfo for _, b := range backends { - if !b.Installed { + if !installedInCluster(b, installed) { continue } hasTag := false diff --git a/core/http/routes/cluster_capabilities.go b/core/http/routes/cluster_capabilities.go index d64db2e13dd0..1b44e89a7bea 100644 --- a/core/http/routes/cluster_capabilities.go +++ b/core/http/routes/cluster_capabilities.go @@ -1,6 +1,8 @@ package routes import ( + "context" + "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/http/endpoints/localai" ) @@ -18,3 +20,33 @@ func ClusterCapabilityProviderFor(app *application.Application) localai.ClusterC } return app.Distributed().Registry.HealthyBackendCapabilities } + +// ClusterInstalledProviderFor returns the install-state source backing every +// backend discovery endpoint that filters on installed backends, or nil in +// single-node mode. +// +// A nil provider leaves those endpoints reading the local filesystem exactly as +// they always have. In distributed mode a backend lives on the worker that runs +// it, so the controller's own disk cannot answer the question; the active +// BackendManager already aggregates the per-node view that GET /backends +// renders, and discovery reuses it rather than growing a second path. +func ClusterInstalledProviderFor(app *application.Application) localai.ClusterInstalledProvider { + if app == nil || !app.IsDistributed() || app.GalleryService() == nil { + return nil + } + return func(ctx context.Context) ([]string, error) { + manager := app.GalleryService().BackendManager() + if manager == nil { + return nil, nil + } + backends, err := manager.ListBackends() + if err != nil { + return nil, err + } + names := make([]string, 0, len(backends)) + for name := range backends { + names = append(names, name) + } + return names, nil + } +} diff --git a/core/http/routes/finetuning.go b/core/http/routes/finetuning.go index 4d3de876f418..a122798787ac 100644 --- a/core/http/routes/finetuning.go +++ b/core/http/routes/finetuning.go @@ -29,7 +29,7 @@ func RegisterFineTuningRoutes(e *echo.Echo, ftService *finetune.FineTuneService, } ft := e.Group("/api/fine-tuning", readyMw, fineTuningMw) - ft.GET("/backends", localai.ListFineTuneBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app))) + ft.GET("/backends", localai.ListFineTuneBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app))) ft.POST("/jobs", localai.StartFineTuneJobEndpoint(ftService)) ft.GET("/jobs", localai.ListFineTuneJobsEndpoint(ftService)) ft.GET("/jobs/:id", localai.GetFineTuneJobEndpoint(ftService)) diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 3745dc46823f..d0e44f8e1ae8 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -64,7 +64,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(appConfig.SystemState), adminMiddleware) router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware) router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware) - router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState, ClusterCapabilityProviderFor(app)), adminMiddleware) + router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app)), adminMiddleware) router.GET("/backends/known", backendGalleryEndpointService.ListKnownBackendsEndpoint(appConfig.SystemState), adminMiddleware) router.GET("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint(), adminMiddleware) router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware) diff --git a/core/http/routes/quantization.go b/core/http/routes/quantization.go index 16eafcd435f9..92f5a15218f0 100644 --- a/core/http/routes/quantization.go +++ b/core/http/routes/quantization.go @@ -29,7 +29,7 @@ func RegisterQuantizationRoutes(e *echo.Echo, qService *quantization.Quantizatio } q := e.Group("/api/quantization", readyMw, quantizationMw) - q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app))) + q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app))) q.POST("/jobs", localai.StartQuantizationJobEndpoint(qService)) q.GET("/jobs", localai.ListQuantizationJobsEndpoint(qService)) q.GET("/jobs/:id", localai.GetQuantizationJobEndpoint(qService))