Skip to content

Commit 45f39f8

Browse files
lmicciniclaude
lmiccini
and
claude
committed
Add per-node secret rotation tracking with drift detection
Implements persistent tracking of secret versions deployed to each node in OpenStackDataPlaneNodeSet to coordinate safe deletion of old credentials during gradual rollouts. Implementation: - ConfigMap-based storage (`<nodeset-name>-secret-tracking`) records which secret versions are deployed to each node - Tracks "Current" (deployed) vs "Expected" (cluster) secret states: - Current: Hash of secrets actually deployed to nodes - Expected: Hash of secrets currently in cluster - Drift detected when Current != Expected - Deployment processing updates tracking data per node with secret hashes, skipping stale deployments (hash != cluster hash) - Drift detection runs after each reconciliation, comparing cluster secrets against tracking ConfigMap, using APIReader to bypass cache - Status field SecretDeployment reports: - UpdatedNodes: count of nodes on current secret versions - AllNodesUpdated: whether all nodes have current versions - ConfigMapName, TotalNodes, LastUpdateTime - APIReader field added to reconciler to read directly from Kubernetes API, bypassing controller-runtime cache for accurate drift detection This enables safe credential deletion only when all nodes across all nodesets sharing the credentials have been updated. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 64e838e commit 45f39f8

12 files changed

Lines changed: 3833 additions & 20 deletions

‎api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1987,6 +1987,35 @@ spec:
19871987
generation, then the controller has not processed the latest changes.
19881988
format: int64
19891989
type: integer
1990+
secretDeployment:
1991+
description: |-
1992+
SecretDeployment tracks secret deployment progress across nodeset nodes
1993+
Details are stored in a ConfigMap to avoid bloating the CR status
1994+
properties:
1995+
allNodesUpdated:
1996+
description: AllNodesUpdated indicates all nodes have current
1997+
versions of all tracked secrets
1998+
type: boolean
1999+
configMapName:
2000+
description: ConfigMapName references the ConfigMap containing
2001+
detailed per-secret tracking
2002+
type: string
2003+
lastUpdateTime:
2004+
description: LastUpdateTime is when this status was last updated
2005+
format: date-time
2006+
type: string
2007+
totalNodes:
2008+
description: TotalNodes is the total number of nodes in the nodeset
2009+
type: integer
2010+
updatedNodes:
2011+
description: UpdatedNodes is count of nodes with all current secret
2012+
versions
2013+
type: integer
2014+
required:
2015+
- allNodesUpdated
2016+
- totalNodes
2017+
- updatedNodes
2018+
type: object
19902019
secretHashes:
19912020
additionalProperties:
19922021
type: string

‎api/dataplane/v1beta1/openstackdataplanenodeset_types.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,28 @@ type OpenStackDataPlaneNodeSetStatus struct {
163163

164164
//DeployedBmhHash - Hash of BMHs deployed
165165
DeployedBmhHash string `json:"deployedBmhHash,omitempty"`
166+
167+
// SecretDeployment tracks secret deployment progress across nodeset nodes
168+
// Details are stored in a ConfigMap to avoid bloating the CR status
169+
SecretDeployment *SecretDeploymentStatus `json:"secretDeployment,omitempty"`
170+
}
171+
172+
// SecretDeploymentStatus tracks secret deployment progress across nodeset nodes
173+
type SecretDeploymentStatus struct {
174+
// AllNodesUpdated indicates all nodes have current versions of all tracked secrets
175+
AllNodesUpdated bool `json:"allNodesUpdated"`
176+
177+
// TotalNodes is the total number of nodes in the nodeset
178+
TotalNodes int `json:"totalNodes"`
179+
180+
// UpdatedNodes is count of nodes with all current secret versions
181+
UpdatedNodes int `json:"updatedNodes"`
182+
183+
// ConfigMapName references the ConfigMap containing detailed per-secret tracking
184+
ConfigMapName string `json:"configMapName,omitempty"`
185+
186+
// LastUpdateTime is when this status was last updated
187+
LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"`
166188
}
167189

168190
// +kubebuilder:object:root=true
@@ -183,6 +205,14 @@ func (instance OpenStackDataPlaneNodeSet) IsReady() bool {
183205
return instance.Status.Conditions.IsTrue(condition.ReadyCondition)
184206
}
185207

208+
// AreAllNodesUpdated returns true if all nodes in the nodeset have been updated
209+
// with current versions of all tracked secrets. Returns false if secret deployment
210+
// tracking is not initialized or if any nodes have pending updates.
211+
func (instance OpenStackDataPlaneNodeSet) AreAllNodesUpdated() bool {
212+
return instance.Status.SecretDeployment != nil &&
213+
instance.Status.SecretDeployment.AllNodesUpdated
214+
}
215+
186216
// InitConditions - Initializes Status Conditons
187217
func (instance *OpenStackDataPlaneNodeSet) InitConditions() {
188218
instance.Status.Conditions = condition.Conditions{}

‎api/dataplane/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindata/crds/crds.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21193,6 +21193,35 @@ spec:
2119321193
generation, then the controller has not processed the latest changes.
2119421194
format: int64
2119521195
type: integer
21196+
secretDeployment:
21197+
description: |-
21198+
SecretDeployment tracks secret deployment progress across nodeset nodes
21199+
Details are stored in a ConfigMap to avoid bloating the CR status
21200+
properties:
21201+
allNodesUpdated:
21202+
description: AllNodesUpdated indicates all nodes have current
21203+
versions of all tracked secrets
21204+
type: boolean
21205+
configMapName:
21206+
description: ConfigMapName references the ConfigMap containing
21207+
detailed per-secret tracking
21208+
type: string
21209+
lastUpdateTime:
21210+
description: LastUpdateTime is when this status was last updated
21211+
format: date-time
21212+
type: string
21213+
totalNodes:
21214+
description: TotalNodes is the total number of nodes in the nodeset
21215+
type: integer
21216+
updatedNodes:
21217+
description: UpdatedNodes is count of nodes with all current secret
21218+
versions
21219+
type: integer
21220+
required:
21221+
- allNodesUpdated
21222+
- totalNodes
21223+
- updatedNodes
21224+
type: object
2119621225
secretHashes:
2119721226
additionalProperties:
2119821227
type: string

‎bindata/rbac/infra-operator-rbac.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@ rules:
159159
- get
160160
- list
161161
- watch
162+
- apiGroups:
163+
- dataplane.openstack.org
164+
resources:
165+
- openstackdataplanenodesets
166+
verbs:
167+
- get
168+
- list
169+
- watch
162170
- apiGroups:
163171
- frrk8s.metallb.io
164172
resources:

‎bindata/rbac/rbac.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,14 @@ rules:
728728
- patch
729729
- update
730730
- watch
731+
- apiGroups:
732+
- rabbitmq.openstack.org
733+
resources:
734+
- rabbitmqusers
735+
verbs:
736+
- get
737+
- list
738+
- watch
731739
- apiGroups:
732740
- rbac.authorization.k8s.io
733741
resources:

‎cmd/main.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -344,9 +344,10 @@ func main() {
344344
os.Exit(1)
345345
}
346346
if err := (&dataplanecontroller.OpenStackDataPlaneNodeSetReconciler{
347-
Client: mgr.GetClient(),
348-
Scheme: mgr.GetScheme(),
349-
Kclient: kclient,
347+
Client: mgr.GetClient(),
348+
APIReader: mgr.GetAPIReader(),
349+
Scheme: mgr.GetScheme(),
350+
Kclient: kclient,
350351
}).SetupWithManager(ctx, mgr); err != nil {
351352
setupLog.Error(err, "unable to create controller", "controller", "OpenStackDataPlaneNodeSet")
352353
os.Exit(1)

‎config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1987,6 +1987,35 @@ spec:
19871987
generation, then the controller has not processed the latest changes.
19881988
format: int64
19891989
type: integer
1990+
secretDeployment:
1991+
description: |-
1992+
SecretDeployment tracks secret deployment progress across nodeset nodes
1993+
Details are stored in a ConfigMap to avoid bloating the CR status
1994+
properties:
1995+
allNodesUpdated:
1996+
description: AllNodesUpdated indicates all nodes have current
1997+
versions of all tracked secrets
1998+
type: boolean
1999+
configMapName:
2000+
description: ConfigMapName references the ConfigMap containing
2001+
detailed per-secret tracking
2002+
type: string
2003+
lastUpdateTime:
2004+
description: LastUpdateTime is when this status was last updated
2005+
format: date-time
2006+
type: string
2007+
totalNodes:
2008+
description: TotalNodes is the total number of nodes in the nodeset
2009+
type: integer
2010+
updatedNodes:
2011+
description: UpdatedNodes is count of nodes with all current secret
2012+
versions
2013+
type: integer
2014+
required:
2015+
- allNodesUpdated
2016+
- totalNodes
2017+
- updatedNodes
2018+
type: object
19902019
secretHashes:
19912020
additionalProperties:
19922021
type: string

‎config/rbac/role.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,14 @@ rules:
679679
- patch
680680
- update
681681
- watch
682+
- apiGroups:
683+
- rabbitmq.openstack.org
684+
resources:
685+
- rabbitmqusers
686+
verbs:
687+
- get
688+
- list
689+
- watch
682690
- apiGroups:
683691
- rbac.authorization.k8s.io
684692
resources:

0 commit comments

Comments
 (0)