Add PreallocatedMode for predetermined GUID lookup

pkg/config/config.go

@@ -27,6 +27,13 @@ type DaemonConfig struct {
 	// with UTC timestamps from GetServerTime().
 	// Example: "America/New_York", "America/Los_Angeles", "UTC" (default).
 	SMTimezone string `env:"SM_TIMEZONE" envDefault:"UTC"`
+	// Use deterministic pre-allocated GUIDs (lookup) instead of allocating from the pool.
+	// Enabled on substrates where we are not the UFM fabric admin and GUIDs/PKeys are
+	// pre-provisioned by the provider (e.g. IREN B300s).
+	PreallocatedMode bool `env:"PREALLOCATED_MODE" envDefault:"false"`
+	// Bits reserved per pkey band in pre-allocated mode; each pkey owns 2^bits guids
+	// (16 = 65536). Must match the provider's per-pkey blocks.
+	PreallocatedBandBits int `env:"PREALLOCATED_BAND_BITS" envDefault:"16"`
 }
 
 type GUIDPoolConfig struct {

pkg/daemon/daemon.go

@@ -500,18 +500,41 @@ func (d *daemon) processNetworkGUID(networkID string, spec *utils.IbSriovCniSpec
 			return err
 		}
 	} else {
-		guidAddr, err = d.guidPool.GenerateGUID()
-		// TODO(Nik): This error handling is funky. We sync the GUID pool but then don't
-		// re-assign a guidAddr so we're left with a 0 guid
-		if err != nil {
-			switch {
-			case errors.Is(err, guid.ErrGUIDPoolExhausted):
-				err = syncGUIDPool(d.smClient, d.guidPool)
-				if err != nil {
-					return err
+		if d.config.PreallocatedMode {
+			guidAddr, err = d.LookupPredeterminedGUID(spec)
+			if err != nil {
+				switch {
+				case errors.Is(err, guid.ErrGUIDPoolExhausted):
+					// Band looks full: it may be stale state. Rebuild the pool from pod
+					// annotations (drift recovery) and retry before giving up.
+					log.Error().Msgf("pre-allocated GUID band exhausted for pod %s (pkey %s); resyncing pool from pods",
+						pi.pod.UID, spec.PKey)
+					if syncErr := d.syncGUIDPoolFromPods(); syncErr != nil {
+						return fmt.Errorf("failed to resync GUID pool from pods for pod %s: %v", pi.pod.UID, syncErr)
+					}
+					guidAddr, err = d.LookupPredeterminedGUID(spec)
+					if err != nil {
+						return fmt.Errorf(
+							"failed to lookup predetermined GUID for pod ID %s after resync: %v", pi.pod.UID, err)
+					}
+				default:
+					return fmt.Errorf("failed to lookup predetermined GUID for pod ID %s, with error: %v", pi.pod.UID, err)
+				}
+			}
+		} else {
+			guidAddr, err = d.guidPool.GenerateGUID()
+			// TODO(Nik): This error handling is funky. We sync the GUID pool but then don't
+			// re-assign a guidAddr so we're left with a 0 guid
+			if err != nil {
+				switch {
+				case errors.Is(err, guid.ErrGUIDPoolExhausted):
+					err = syncGUIDPool(d.smClient, d.guidPool)
+					if err != nil {
+						return err
+					}
+				default:
+					return fmt.Errorf("failed to generate GUID for pod ID %s, with error: %v", pi.pod.UID, err)
 				}
-			default:
-				return fmt.Errorf("failed to generate GUID for pod ID %s, with error: %v", pi.pod.UID, err)
 			}
 		}
 
@@ -540,6 +563,47 @@ func (d *daemon) processNetworkGUID(networkID string, spec *utils.IbSriovCniSpec
 	return nil
 }
 
+// LookupPredeterminedGUID returns a GUID from the pre-allocated band the pod's pkey owns.
+func (d *daemon) LookupPredeterminedGUID(spec *utils.IbSriovCniSpec) (guid.GUID, error) {
+	pKey, err := utils.ParsePKey(spec.PKey)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse pkey %q: %v", spec.PKey, err)
+	}
+
+	guidAddr, err := d.guidPool.GenerateGUIDForPKey(pKey, d.config.PreallocatedBandBits)
+	if err != nil {
+		return 0, fmt.Errorf("failed to allocate GUID in pkey %d band: %w", pKey, err)
+	}
+	return guidAddr, nil
+}
+
+// syncGUIDPoolFromPods rebuilds the pool's in-use set from GUIDs on pod network annotations,
+// without querying UFM. Used in pre-allocated mode to recover from drift on exhaustion.
+func (d *daemon) syncGUIDPoolFromPods() error {
+	pods, err := d.kubeClient.GetPods(kapi.NamespaceAll)
+	if err != nil {
+		return err
+	}
+
+	var usedGuids []string
+	for i := range pods.Items {
+		networks, err := netAttUtils.ParsePodNetworkAnnotation(&pods.Items[i])
+		if err != nil {
+			continue
+		}
+		for _, network := range networks {
+			if !utils.IsPodNetworkConfiguredWithInfiniBand(network) {
+				continue
+			}
+			if g, err := utils.GetPodNetworkGUID(network); err == nil {
+				usedGuids = append(usedGuids, g)
+			}
+		}
+	}
+
+	return d.guidPool.Reset(usedGuids)
+}
+
 func syncGUIDPool(smClient plugins.SubnetManagerClient, guidPool guid.Pool) error {
 	usedGuids, err := smClient.ListGuidsInUse()
 	if err != nil {

pkg/guid/guid_pool.go

@@ -17,6 +17,10 @@ type Pool interface {
 
 	GenerateGUID() (GUID, error)
 
+	// GenerateGUIDForPKey returns the next free guid within the band that the given pkey
+	// owns (used in pre-allocated mode). It does not mark the guid allocated.
+	GenerateGUIDForPKey(pkey, bandBits int) (GUID, error)
+
 	// ReleaseGUID release the reservation of the guid.
 	// It returns error if the guid is not in the range.
 	ReleaseGUID(string) error
@@ -111,6 +115,25 @@ func (p *guidPool) GenerateGUID() (GUID, error) {
 	return 0, ErrGUIDPoolExhausted
 }
 
+// GenerateGUIDForPKey returns the next free guid in the pkey's band
+// (base = rangeStart + pkey*2^bandBits), without marking it allocated.
+func (p *guidPool) GenerateGUIDForPKey(pkey, bandBits int) (GUID, error) {
+	if pkey < 0 {
+		return 0, fmt.Errorf("invalid pkey %d", pkey)
+	}
+	bandSize := uint64(1) << uint(bandBits)
+	base := uint64(p.rangeStart) + uint64(pkey)*bandSize
+	end := base + bandSize - 1
+	if GUID(end) > p.rangeEnd {
+		return 0, fmt.Errorf("pkey %d band [0x%x-0x%x] exceeds pool range end %v", pkey, base, end, p.rangeEnd)
+	}
+
+	if g := p.getFreeGUID(GUID(base), GUID(end)); g != 0 {
+		return g, nil
+	}
+	return 0, ErrGUIDPoolExhausted
+}
+
 // ReleaseGUID release allocated guid
 func (p *guidPool) ReleaseGUID(guid string) error {
 	log.Debug().Msgf("releasing guid %s", guid)

pkg/guid/guid_test.go

@@ -214,4 +214,43 @@ var _ = Describe("GUID Pool", func() {
 			Expect(err).To(HaveOccurred())
 		})
 	})
+	Context("GenerateGUIDForPKey", func() {
+		It("returns the first guid in the pkey's band", func() {
+			pool, err := NewPool(conf)
+			Expect(err).ToNot(HaveOccurred())
+			g, err := pool.GenerateGUIDForPKey(2, 4)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(g.String()).To(Equal("02:00:00:00:00:00:00:20"))
+		})
+		It("hands out the next free slot within the band", func() {
+			pool, err := NewPool(conf)
+			Expect(err).ToNot(HaveOccurred())
+			g, err := pool.GenerateGUIDForPKey(2, 4)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pool.AllocateGUID(g.String())).ToNot(HaveOccurred())
+			g2, err := pool.GenerateGUIDForPKey(2, 4)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(g2.String()).To(Equal("02:00:00:00:00:00:00:21"))
+		})
+		It("gives different pkeys non-overlapping bands", func() {
+			pool, err := NewPool(conf)
+			Expect(err).ToNot(HaveOccurred())
+			g2, err := pool.GenerateGUIDForPKey(2, 4)
+			Expect(err).ToNot(HaveOccurred())
+			g3, err := pool.GenerateGUIDForPKey(3, 4)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(g2.String()).To(Equal("02:00:00:00:00:00:00:20"))
+			Expect(g3.String()).To(Equal("02:00:00:00:00:00:00:30"))
+		})
+		It("fails when the pkey band exceeds the pool range", func() {
+			smallConf := &config.GUIDPoolConfig{RangeStart: "00:00:00:00:00:00:01:00",
+				RangeEnd: "00:00:00:00:00:00:01:0f"}
+			pool, err := NewPool(smallConf)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = pool.GenerateGUIDForPKey(0, 4)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = pool.GenerateGUIDForPKey(1, 4)
+			Expect(err).To(HaveOccurred())
+		})
+	})
 })