|
| 1 | +package gallery_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "os" |
| 5 | + "path/filepath" |
| 6 | + |
| 7 | + "github.com/mudler/LocalAI/core/config" |
| 8 | + . "github.com/mudler/LocalAI/core/gallery" |
| 9 | + "github.com/mudler/LocalAI/pkg/system" |
| 10 | + . "github.com/onsi/ginkgo/v2" |
| 11 | + . "github.com/onsi/gomega" |
| 12 | + "gopkg.in/yaml.v3" |
| 13 | +) |
| 14 | + |
| 15 | +// These specs cover backend discovery in distributed mode, where the machine |
| 16 | +// answering GET /backends/available (the controller) is not the machine that |
| 17 | +// will run the backend (a worker). Filtering the listing against the |
| 18 | +// controller's own hardware hid every GPU-only meta backend from admins. |
| 19 | +var _ = Describe("AvailableBackendsForCapabilities", func() { |
| 20 | + var ( |
| 21 | + tempDir string |
| 22 | + galleryPath string |
| 23 | + galleries []config.Gallery |
| 24 | + // controller stands in for a GPU-less frontend pod: no vendor, so its |
| 25 | + // reported capability is "default". |
| 26 | + controller *system.SystemState |
| 27 | + ) |
| 28 | + |
| 29 | + writeGalleryYAML := func(backends []GalleryBackend) { |
| 30 | + data, err := yaml.Marshal(backends) |
| 31 | + Expect(err).NotTo(HaveOccurred()) |
| 32 | + Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed()) |
| 33 | + } |
| 34 | + |
| 35 | + names := func(backends GalleryElements[*GalleryBackend]) []string { |
| 36 | + out := []string{} |
| 37 | + for _, b := range backends { |
| 38 | + out = append(out, b.GetName()) |
| 39 | + } |
| 40 | + return out |
| 41 | + } |
| 42 | + |
| 43 | + // longcatVideo mirrors the real gallery entry that triggered this bug: a |
| 44 | + // meta backend enumerating only NVIDIA variants, with neither a "default" |
| 45 | + // nor a "cpu" key for Capability() to fall back to. |
| 46 | + longcatVideo := GalleryBackend{ |
| 47 | + Metadata: Metadata{Name: "longcat-video"}, |
| 48 | + CapabilitiesMap: map[string]string{ |
| 49 | + "nvidia": "longcat-video-nvidia", |
| 50 | + "nvidia-cuda-12": "longcat-video-cuda-12", |
| 51 | + "nvidia-cuda-13": "longcat-video-cuda-13", |
| 52 | + "nvidia-l4t-cuda-13": "longcat-video-l4t-cuda-13", |
| 53 | + }, |
| 54 | + } |
| 55 | + |
| 56 | + // cpuMeta is compatible with every host and must keep showing up in all |
| 57 | + // scenarios, proving the union widens the listing without replacing it. |
| 58 | + cpuMeta := GalleryBackend{ |
| 59 | + Metadata: Metadata{Name: "whisper"}, |
| 60 | + CapabilitiesMap: map[string]string{"default": "whisper-cpu", "nvidia": "whisper-cuda"}, |
| 61 | + } |
| 62 | + |
| 63 | + BeforeEach(func() { |
| 64 | + var err error |
| 65 | + tempDir, err = os.MkdirTemp("", "node-capability-test-*") |
| 66 | + Expect(err).NotTo(HaveOccurred()) |
| 67 | + DeferCleanup(func() { |
| 68 | + Expect(os.RemoveAll(tempDir)).To(Succeed()) |
| 69 | + }) |
| 70 | + |
| 71 | + galleryPath = filepath.Join(tempDir, "gallery.yaml") |
| 72 | + writeGalleryYAML([]GalleryBackend{longcatVideo, cpuMeta}) |
| 73 | + |
| 74 | + galleries = []config.Gallery{{Name: "test-gallery", URL: "file://" + galleryPath}} |
| 75 | + controller = system.NewCapabilityState("default", system.WithBackendPath(tempDir)) |
| 76 | + }) |
| 77 | + |
| 78 | + It("hides a GPU-only meta when no node capabilities are supplied", func() { |
| 79 | + backends, err := AvailableBackendsForCapabilities(galleries, controller, nil) |
| 80 | + Expect(err).NotTo(HaveOccurred()) |
| 81 | + Expect(names(backends)).To(ContainElement("whisper")) |
| 82 | + Expect(names(backends)).NotTo(ContainElement("longcat-video")) |
| 83 | + }) |
| 84 | + |
| 85 | + It("lists a GPU-only meta runnable on a registered worker node", func() { |
| 86 | + backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia-l4t-cuda-13"}) |
| 87 | + Expect(err).NotTo(HaveOccurred()) |
| 88 | + Expect(names(backends)).To(ContainElement("longcat-video")) |
| 89 | + Expect(names(backends)).To(ContainElement("whisper")) |
| 90 | + }) |
| 91 | + |
| 92 | + It("unions across heterogeneous nodes rather than intersecting them", func() { |
| 93 | + writeGalleryYAML([]GalleryBackend{ |
| 94 | + longcatVideo, |
| 95 | + { |
| 96 | + Metadata: Metadata{Name: "amd-only"}, |
| 97 | + CapabilitiesMap: map[string]string{"amd": "amd-only-rocm"}, |
| 98 | + }, |
| 99 | + }) |
| 100 | + |
| 101 | + backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia", "amd"}) |
| 102 | + Expect(err).NotTo(HaveOccurred()) |
| 103 | + Expect(names(backends)).To(ContainElements("longcat-video", "amd-only")) |
| 104 | + }) |
| 105 | + |
| 106 | + It("still excludes a meta no node can satisfy", func() { |
| 107 | + backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"amd"}) |
| 108 | + Expect(err).NotTo(HaveOccurred()) |
| 109 | + Expect(names(backends)).NotTo(ContainElement("longcat-video")) |
| 110 | + }) |
| 111 | + |
| 112 | + It("filters concrete backends by node capability too", func() { |
| 113 | + writeGalleryYAML([]GalleryBackend{ |
| 114 | + {Metadata: Metadata{Name: "some-backend-cuda"}, URI: "quay.io/test/cuda"}, |
| 115 | + {Metadata: Metadata{Name: "some-backend-rocm"}, URI: "quay.io/test/rocm"}, |
| 116 | + }) |
| 117 | + |
| 118 | + backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia"}) |
| 119 | + Expect(err).NotTo(HaveOccurred()) |
| 120 | + Expect(names(backends)).To(ContainElement("some-backend-cuda")) |
| 121 | + Expect(names(backends)).NotTo(ContainElement("some-backend-rocm")) |
| 122 | + }) |
| 123 | + |
| 124 | + It("returns exactly the single-node listing when the node list is empty", func() { |
| 125 | + withNodes, err := AvailableBackendsForCapabilities(galleries, controller, []string{}) |
| 126 | + Expect(err).NotTo(HaveOccurred()) |
| 127 | + baseline, err := AvailableBackends(galleries, controller) |
| 128 | + Expect(err).NotTo(HaveOccurred()) |
| 129 | + Expect(names(withNodes)).To(Equal(names(baseline))) |
| 130 | + }) |
| 131 | +}) |
0 commit comments