feat(distributed): per-node backend installation from the gallery

In distributed mode the Backends gallery used to fan every install out to
every worker — fine for auto-resolving (meta) backends like llama-cpp where
each node picks its own variant, but wrong for hardware-specific builds
like cpu-llama-cpp that would silently land on every GPU node.

Adds a node-targeted install path through the existing
POST /api/nodes/:id/backends/install plumbing, with two entry points:

- Backends gallery row gets a split-button in distributed mode. Auto-
  resolving keeps "Install on all nodes" as the primary; chevron menu
  opens the picker. Hardware-specific routes the primary directly to the
  picker — no fan-out path on the row.
- Nodes-page drawer gets a "+ Add backend" button that navigates to
  /app/backends?target=<node-id>; the gallery scopes itself to that node
  (banner, single per-row install button, Reinstall/Remove for already-
  installed). One gallery, two scopes — no second UI to maintain.

The picker (new NodeInstallPicker) shows a 3-state suitability column
(Compatible / Override / Installed), an auto-expanding variant override
disclosure that fires when selected nodes have no working GPU, parallel
per-node installs with inline status and Retry-failed-nodes, and a
mismatch confirm that names the consequence on the button itself.

A 409 fan-out guard on /api/backends/apply protects CLI/Terraform/script
users from the same footgun: hardware-specific installs in distributed
mode now return code "concrete_backend_requires_target" with a human-
readable error and a meta_alternative pointer.

The gallery list payload now surfaces capabilities, metaBackendFor and
per-row nodes (NodeBackendRef) so the picker and the new Nodes column
have everything they need without re-walking the gallery client-side.

GODEBUG=netdns=go is set on the compose services because the cgo DNS
resolver follows the container's nsswitch.conf to host systemd-resolved
(127.0.0.53), unreachable from inside the container; the pure-Go
resolver reads /etc/resolv.conf directly and uses Docker's embedded DNS.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude Code:claude-opus-4-7[1m] [Edit] [Bash] [Read] [Write]

Author: mudler

core/http/endpoints/localai/backend.go

@@ -98,14 +98,26 @@ func (mgs *BackendEndpointService) GetAllStatusEndpoint() echo.HandlerFunc {
 // @Param request body GalleryBackend true "query params"
 // @Success 200 {object} schema.BackendResponse "Response"
 // @Router /backends/apply [post]
-func (mgs *BackendEndpointService) ApplyBackendEndpoint() echo.HandlerFunc {
+func (mgs *BackendEndpointService) ApplyBackendEndpoint(systemState *system.SystemState) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		input := new(GalleryBackend)
 		// Get input data from the request body
 		if err := c.Bind(input); err != nil {
 			return err
 		}
 
+		// In distributed mode, refuse to fan out a hardware-specific build to
+		// every node — a CPU build landing on a GPU cluster is almost always
+		// wrong, and the silent footgun is exactly what this guard exists for.
+		// Auto-resolving (meta) backends are fine because each node picks its
+		// own variant. Tooling can recover by hitting
+		// POST /api/nodes/{id}/backends/install per target node.
+		if mgs.backendApplier.BackendManager().IsDistributed() && input.ID != "" {
+			if guard := concreteFanOutGuard(c, mgs.galleries, systemState, input.ID); guard != nil {
+				return guard
+			}
+		}
+
 		uuid, err := uuid.NewUUID()
 		if err != nil {
 			return err
@@ -120,6 +132,66 @@ func (mgs *BackendEndpointService) ApplyBackendEndpoint() echo.HandlerFunc {
 	}
 }
 
+// concreteFanOutGuard returns a 409 response if the requested backend is a
+// hardware-specific build (not auto-resolving / meta) and we are in
+// distributed mode. It looks up the backend in the configured galleries; if
+// the lookup itself fails (gallery unreachable, name not found), the guard
+// stays out of the way and lets the install enqueue normally — a missing
+// name will surface from the worker as a clearer error than the guard could
+// produce here. The response body deliberately speaks human, with `code` and
+// `meta_alternative` as the programmatic contract for tooling.
+func concreteFanOutGuard(c echo.Context, galleries []config.Gallery, systemState *system.SystemState, backendID string) error {
+	// Use the unfiltered listing because in distributed mode the frontend's
+	// hardware is irrelevant — the install targets workers, not us — and the
+	// filtered list would hide variants that don't match the frontend host
+	// (e.g. a CUDA build on a CPU-only frontend), preventing the guard from
+	// firing for exactly the cases it's meant to protect against.
+	available, err := gallery.AvailableBackendsUnfiltered(galleries, systemState)
+	if err != nil {
+		return nil
+	}
+	requested := available.FindByName(backendID)
+	if requested == nil || requested.IsMeta() {
+		return nil
+	}
+
+	// Try to find an auto-resolving (meta) backend that has this concrete
+	// variant in its CapabilitiesMap, so we can suggest it as a one-shot
+	// alternative. Optional — empty string is fine if no parent exists.
+	metaAlternative := ""
+	for _, b := range available {
+		if !b.IsMeta() {
+			continue
+		}
+		for _, concrete := range b.CapabilitiesMap {
+			if concrete == backendID {
+				metaAlternative = b.Name
+				break
+			}
+		}
+		if metaAlternative != "" {
+			break
+		}
+	}
+
+	msg := fmt.Sprintf(
+		"Backend %q is a hardware-specific build and won't run correctly on every node in this cluster. In distributed mode, install it on specific nodes:\n\n  POST /api/nodes/{node_id}/backends/install\n  {\"backend\": %q}",
+		backendID, backendID,
+	)
+	if metaAlternative != "" {
+		msg += fmt.Sprintf(
+			"\n\nTo install across all nodes, use the auto-resolving backend %q — each node picks its own variant based on its hardware.",
+			metaAlternative,
+		)
+	}
+
+	return c.JSON(409, map[string]any{
+		"error":            msg,
+		"code":             "concrete_backend_requires_target",
+		"meta_alternative": metaAlternative,
+	})
+}
+
 // DeleteBackendEndpoint lets delete backends from a LocalAI instance
 // @Summary delete backends from LocalAI.
 // @Tags backends

core/http/endpoints/localai/nodes.go

@@ -363,6 +363,9 @@ func ResumeNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
 }
 
 // InstallBackendOnNodeEndpoint triggers backend installation on a worker node via NATS.
+// Backend can be either a gallery ID (resolved against BackendGalleries) or a
+// direct URI install (URI + Name + optional Alias) — same shape as the
+// standalone /api/backends/install-external path, just scoped to one node.
 func InstallBackendOnNodeEndpoint(unloader nodes.NodeCommandSender) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		if unloader == nil {
@@ -372,17 +375,24 @@ func InstallBackendOnNodeEndpoint(unloader nodes.NodeCommandSender) echo.Handler
 		var req struct {
 			Backend          string `json:"backend"`
 			BackendGalleries string `json:"backend_galleries,omitempty"`
+			URI              string `json:"uri,omitempty"`
+			Name             string `json:"name,omitempty"`
+			Alias            string `json:"alias,omitempty"`
 		}
-		if err := c.Bind(&req); err != nil || req.Backend == "" {
-			return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "backend name required"))
+		if err := c.Bind(&req); err != nil {
+			return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "invalid request body"))
+		}
+		// Either a gallery backend name or a direct URI must be supplied.
+		if req.Backend == "" && req.URI == "" {
+			return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "backend name or uri required"))
 		}
-		reply, err := unloader.InstallBackend(nodeID, req.Backend, "", req.BackendGalleries, "", "", "")
+		reply, err := unloader.InstallBackend(nodeID, req.Backend, "", req.BackendGalleries, req.URI, req.Name, req.Alias)
 		if err != nil {
-			xlog.Error("Failed to install backend on node", "node", nodeID, "backend", req.Backend, "error", err)
+			xlog.Error("Failed to install backend on node", "node", nodeID, "backend", req.Backend, "uri", req.URI, "error", err)
 			return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to install backend on node"))
 		}
 		if !reply.Success {
-			xlog.Error("Backend install failed on node", "node", nodeID, "backend", req.Backend, "error", reply.Error)
+			xlog.Error("Backend install failed on node", "node", nodeID, "backend", req.Backend, "uri", req.URI, "error", reply.Error)
 			return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "backend installation failed"))
 		}
 		return c.JSON(http.StatusOK, map[string]string{"message": "backend installed"})