Skip to content

Commit 4e5fd54

Browse files
authored
Merge pull request #1670 from rackerlabs/nautobotOPRacks
feat: Add Rack Sync to Nautobot Operator
2 parents 260d1e8 + fa44c22 commit 4e5fd54

21 files changed

Lines changed: 953 additions & 17 deletions

File tree

go/nautobotop/api/v1alpha1/nautobot_types.go

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,16 @@ type NautobotSpec struct {
2828
// +kubebuilder:default=600
2929
RequeueAfter int `json:"requeueAfter,omitempty"`
3030
// +kubebuilder:default=172800
31-
SyncIntervalSeconds int `json:"syncIntervalSeconds,omitempty"`
32-
NautobotSecretRef SecretKeySelector `json:"nautobotSecretRef,omitempty"`
33-
NautobotServiceRef ServiceSelector `json:"nautobotServiceRef,omitempty"`
34-
DeviceTypesRef []ConfigMapRef `json:"deviceTypeRef,omitempty"`
35-
LocationTypesRef []ConfigMapRef `json:"locationTypesRef,omitempty"`
36-
LocationRef []ConfigMapRef `json:"locationRef,omitempty"`
37-
RackGroupRef []ConfigMapRef `json:"rackGroupRef,omitempty"`
31+
SyncIntervalSeconds int `json:"syncIntervalSeconds,omitempty"`
32+
// +kubebuilder:default=70000
33+
CacheMaxSize int `json:"cacheMaxSize,omitempty"`
34+
NautobotSecretRef SecretKeySelector `json:"nautobotSecretRef,omitempty"`
35+
NautobotServiceRef ServiceSelector `json:"nautobotServiceRef,omitempty"`
36+
DeviceTypesRef []ConfigMapRef `json:"deviceTypeRef,omitempty"`
37+
LocationTypesRef []ConfigMapRef `json:"locationTypesRef,omitempty"`
38+
LocationRef []ConfigMapRef `json:"locationRef,omitempty"`
39+
RackGroupRef []ConfigMapRef `json:"rackGroupRef,omitempty"`
40+
RackRef []ConfigMapRef `json:"rackRef,omitempty"`
3841
}
3942

4043
// NautobotStatus defines the observed state of Nautobot.

go/nautobotop/api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go/nautobotop/config/crd/bases/sync.rax.io_nautobots.yaml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ spec:
3939
spec:
4040
description: NautobotSpec defines the desired state of Nautobot.
4141
properties:
42+
cacheMaxSize:
43+
default: 70000
44+
type: integer
4245
deviceTypeRef:
4346
items:
4447
description: ConfigMapRef defines a reference to a specific ConfigMap
@@ -205,6 +208,35 @@ spec:
205208
- name
206209
type: object
207210
type: array
211+
rackRef:
212+
items:
213+
description: ConfigMapRef defines a reference to a specific ConfigMap
214+
properties:
215+
configMapSelector:
216+
description: The name of the ConfigMap resource being referred
217+
to
218+
properties:
219+
key:
220+
description: The key in the ConfigMap data
221+
type: string
222+
name:
223+
description: The name of the ConfigMap
224+
minLength: 1
225+
type: string
226+
namespace:
227+
description: The namespace where the ConfigMap resides
228+
type: string
229+
required:
230+
- name
231+
type: object
232+
name:
233+
description: Name of this config set (logical name)
234+
type: string
235+
required:
236+
- configMapSelector
237+
- name
238+
type: object
239+
type: array
208240
requeueAfter:
209241
default: 600
210242
type: integer

go/nautobotop/go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.24.0
44

55
require (
66
github.com/charmbracelet/log v0.4.2
7+
github.com/maypok86/otter/v2 v2.3.0
78
github.com/nautobot/go-nautobot/v3 v3.0.0-20241004125816-802218866be0
89
github.com/onsi/ginkgo/v2 v2.22.0
910
github.com/onsi/gomega v1.36.1
@@ -62,6 +63,7 @@ require (
6263
github.com/muesli/termenv v0.16.0 // indirect
6364
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
6465
github.com/pkg/errors v0.9.1 // indirect
66+
github.com/pmezard/go-difflib v1.0.0 // indirect
6567
github.com/prometheus/client_golang v1.22.0 // indirect
6668
github.com/prometheus/client_model v0.6.1 // indirect
6769
github.com/prometheus/common v0.62.0 // indirect
@@ -70,6 +72,7 @@ require (
7072
github.com/spf13/cobra v1.8.1 // indirect
7173
github.com/spf13/pflag v1.0.5 // indirect
7274
github.com/stoewer/go-strcase v1.3.0 // indirect
75+
github.com/stretchr/testify v1.11.1 // indirect
7376
github.com/x448/float16 v0.8.4 // indirect
7477
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
7578
go.opentelemetry.io/auto/sdk v1.1.0 // indirect

go/nautobotop/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
111111
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
112112
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
113113
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
114+
github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=
115+
github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8=
114116
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
115117
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
116118
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

go/nautobotop/helm/crds/clients.yaml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ spec:
3939
spec:
4040
description: NautobotSpec defines the desired state of Nautobot.
4141
properties:
42+
cacheMaxSize:
43+
default: 70000
44+
type: integer
4245
deviceTypeRef:
4346
items:
4447
description: ConfigMapRef defines a reference to a specific ConfigMap
@@ -205,6 +208,35 @@ spec:
205208
- name
206209
type: object
207210
type: array
211+
rackRef:
212+
items:
213+
description: ConfigMapRef defines a reference to a specific ConfigMap
214+
properties:
215+
configMapSelector:
216+
description: The name of the ConfigMap resource being referred
217+
to
218+
properties:
219+
key:
220+
description: The key in the ConfigMap data
221+
type: string
222+
name:
223+
description: The name of the ConfigMap
224+
minLength: 1
225+
type: string
226+
namespace:
227+
description: The namespace where the ConfigMap resides
228+
type: string
229+
required:
230+
- name
231+
type: object
232+
name:
233+
description: Name of this config set (logical name)
234+
type: string
235+
required:
236+
- configMapSelector
237+
- name
238+
type: object
239+
type: array
208240
requeueAfter:
209241
default: 600
210242
type: integer

go/nautobotop/internal/controller/nautobot_controller.go

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ func (r *NautobotReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
8484
{name: "locationType", configRefs: nautobotCR.Spec.LocationTypesRef, syncFunc: r.syncLocationTypes},
8585
{name: "location", configRefs: nautobotCR.Spec.LocationRef, syncFunc: r.syncLocation},
8686
{name: "rackGroup", configRefs: nautobotCR.Spec.RackGroupRef, syncFunc: r.syncRackGroup},
87+
{name: "rack", configRefs: nautobotCR.Spec.RackRef, syncFunc: r.syncRack},
8788
{name: "deviceType", configRefs: nautobotCR.Spec.DeviceTypesRef, syncFunc: r.syncDeviceTypes},
8889
}
8990

@@ -125,7 +126,19 @@ func (r *NautobotReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
125126
return ctrl.Result{}, err
126127
}
127128
nautobotURL := fmt.Sprintf("http://%s.%s.svc.cluster.local/api", nautobotCR.Spec.NautobotServiceRef.Name, nautobotCR.Spec.NautobotServiceRef.Namespace)
128-
nautobotClient := nbClient.NewNautobotClient(nautobotURL, username, token)
129+
nautobotClient, err := nbClient.NewNautobotClient(nautobotURL, username, token, nautobotCR.Spec.CacheMaxSize)
130+
if err != nil {
131+
log.Error(err, "failed to create nautobot client")
132+
return ctrl.Result{}, err
133+
}
134+
135+
if err := nautobotClient.PreLoadCacheForLookup(ctx); err != nil {
136+
log.Error(err, "failed to warmup cache")
137+
}
138+
defer func() {
139+
nautobotClient.Cache.Clear()
140+
log.Info("cleared cache after reconcile")
141+
}()
129142

130143
// Sync resources that need updating
131144
for _, res := range resources {
@@ -205,6 +218,19 @@ func (r *NautobotReconciler) syncRackGroup(ctx context.Context,
205218
return nil
206219
}
207220

221+
func (r *NautobotReconciler) syncRack(ctx context.Context,
222+
nautobotClient *nbClient.NautobotClient,
223+
rackData map[string]string,
224+
) error {
225+
log := logf.FromContext(ctx)
226+
log.Info("syncing racks", "count", len(rackData))
227+
syncSvc := sync.NewRackSync(nautobotClient)
228+
if err := syncSvc.SyncAll(ctx, rackData); err != nil {
229+
return fmt.Errorf("failed to sync racks: %w", err)
230+
}
231+
return nil
232+
}
233+
208234
func (r *NautobotReconciler) syncLocation(ctx context.Context,
209235
nautobotClient *nbClient.NautobotClient,
210236
locationType map[string]string,
Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package cache
2+
3+
import (
4+
"fmt"
5+
"time"
6+
7+
"github.com/maypok86/otter/v2"
8+
)
9+
10+
const DEFAULT_CACHE_SIZE = 70_000
11+
12+
// Cache provides a simple key-value cache for Nautobot resources
13+
type Cache struct {
14+
store *otter.Cache[string, any]
15+
}
16+
17+
// New creates a new cache instance
18+
// New creates a new cache instance with the specified maximum size
19+
func New(maxSize int) (*Cache, error) {
20+
if maxSize <= 0 {
21+
maxSize = DEFAULT_CACHE_SIZE
22+
}
23+
24+
store, err := otter.New(&otter.Options[string, any]{
25+
MaximumSize: maxSize,
26+
ExpiryCalculator: otter.ExpiryWriting[string, any](time.Hour),
27+
})
28+
if err != nil {
29+
return nil, fmt.Errorf("failed to create cache: %w", err)
30+
}
31+
32+
return &Cache{store: store}, nil
33+
}
34+
35+
// Get retrieves a value from cache by key
36+
func (c *Cache) Get(key string) (any, bool) {
37+
return c.store.GetIfPresent(key)
38+
}
39+
40+
// Set stores a value in cache with the given key
41+
func (c *Cache) Set(key string, value any) {
42+
c.store.Set(key, value)
43+
}
44+
45+
// SetCollection stores a collection of items in cache
46+
func (c *Cache) SetCollection(resourceType string, items any) {
47+
c.Set(BuildKey(resourceType), items)
48+
}
49+
50+
// FindByName searches a cached collection for an item by name
51+
func FindByName[T any](c *Cache, resourceType, name string, getName func(T) string) (T, bool) {
52+
var zero T
53+
val, ok := c.Get(BuildKey(resourceType))
54+
if !ok {
55+
return zero, false
56+
}
57+
58+
items, ok := val.([]T)
59+
if !ok {
60+
return zero, false
61+
}
62+
63+
for _, item := range items {
64+
if getName(item) == name {
65+
return item, true
66+
}
67+
}
68+
return zero, false
69+
}
70+
71+
// FindByID searches a cached collection for an item by ID
72+
func FindByID[T any](c *Cache, resourceType, id string, getID func(T) *string) (T, bool) {
73+
var zero T
74+
val, ok := c.Get(BuildKey(resourceType))
75+
if !ok {
76+
return zero, false
77+
}
78+
79+
items, ok := val.([]T)
80+
if !ok {
81+
return zero, false
82+
}
83+
84+
for _, item := range items {
85+
itemID := getID(item)
86+
if itemID != nil && *itemID == id {
87+
return item, true
88+
}
89+
}
90+
return zero, false
91+
}
92+
93+
// AddToCollection adds a new item to a cached collection
94+
func AddToCollection[T any](c *Cache, resourceType string, item T) {
95+
key := BuildKey(resourceType)
96+
val, ok := c.Get(key)
97+
if !ok {
98+
c.Set(key, []T{item})
99+
return
100+
}
101+
102+
items, ok := val.([]T)
103+
if !ok {
104+
c.Set(key, []T{item})
105+
return
106+
}
107+
108+
items = append(items, item)
109+
c.Set(key, items)
110+
}
111+
112+
// UpdateInCollection updates an existing item in a cached collection
113+
func UpdateInCollection[T any](c *Cache, resourceType string, updatedItem T, matchFunc func(T) bool) {
114+
key := BuildKey(resourceType)
115+
val, ok := c.Get(key)
116+
if !ok {
117+
return
118+
}
119+
120+
items, ok := val.([]T)
121+
if !ok {
122+
return
123+
}
124+
125+
// Find and update the item
126+
for i, item := range items {
127+
if matchFunc(item) {
128+
items[i] = updatedItem
129+
c.Set(key, items)
130+
return
131+
}
132+
}
133+
}
134+
135+
// RemoveFromCollection removes an item from a cached collection
136+
func RemoveFromCollection[T any](c *Cache, resourceType string, matchFunc func(T) bool) {
137+
key := BuildKey(resourceType)
138+
val, ok := c.Get(key)
139+
if !ok {
140+
return
141+
}
142+
143+
items, ok := val.([]T)
144+
if !ok {
145+
return
146+
}
147+
148+
// Filter out the item
149+
filtered := make([]T, 0, len(items))
150+
for _, item := range items {
151+
if !matchFunc(item) {
152+
filtered = append(filtered, item)
153+
}
154+
}
155+
c.Set(key, filtered)
156+
}
157+
158+
// Delete removes a value from cache by key
159+
func (c *Cache) Delete(key string) {
160+
c.store.Invalidate(key)
161+
}
162+
163+
// Clear removes all entries from the cache
164+
func (c *Cache) Clear() {
165+
c.store.InvalidateAll()
166+
}
167+
168+
// Close closes the cache and releases resources
169+
func (c *Cache) Close() {
170+
c.store.StopAllGoroutines()
171+
}
172+
173+
// BuildKey creates a cache key for a resource collection
174+
func BuildKey(resourceType string) string {
175+
return resourceType
176+
}

0 commit comments

Comments
 (0)