Skip to content

Commit 3074ead

Browse files
DanNiEShclaude
andcommitted
Add automatic reconnect on Ironic authentication failure
When gophercloud's built-in token refresh (ReauthFunc) fails — e.g. due to a network interruption or Keystone being temporarily unavailable — the Ironic client was left permanently stuck with a stale token until the pod restarted. This adds a fallback that detects 401/reauth errors, recreates the service client from scratch, and retries the operation once. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 3ad9e1a commit 3074ead

2 files changed

Lines changed: 213 additions & 5 deletions

File tree

internal/ironic/client.go

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ package ironic
1919

2020
import (
2121
"context"
22+
"errors"
2223
"fmt"
24+
"log/slog"
25+
"net/http"
2326

2427
"github.com/gophercloud/gophercloud/v2"
2528
"github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes"
@@ -36,7 +39,8 @@ type NodeClient interface {
3639
// Client talks to Ironic over REST via gophercloud.
3740
type (
3841
Client struct {
39-
serviceClient *gophercloud.ServiceClient
42+
serviceClient *gophercloud.ServiceClient
43+
newServiceClient func() (*gophercloud.ServiceClient, error)
4044
}
4145

4246
TargetPowerState struct {
@@ -56,24 +60,69 @@ var (
5660
SoftRebooting = TargetPowerState{nodes.SoftRebooting}
5761
)
5862

59-
// NewClient creates an Ironic client with no-auth (typical for in-cluster Ironic with no Keystone).
63+
// NewClient creates an Ironic client configured via clouds.yaml or OS_* environment variables.
6064
func NewClient() (*Client, error) {
61-
client, err := clientconfig.NewServiceClient(context.TODO(), "baremetal", nil)
65+
factory := func() (*gophercloud.ServiceClient, error) {
66+
return clientconfig.NewServiceClient(context.TODO(), "baremetal", nil)
67+
}
68+
sc, err := factory()
6269
if err != nil {
6370
return nil, fmt.Errorf("failed to create baremetal client: %w", err)
6471
}
65-
if client.Endpoint == "" {
72+
if sc.Endpoint == "" {
6673
return nil, fmt.Errorf("baremetal client has no endpoint configured")
6774
}
6875
return &Client{
69-
serviceClient: client,
76+
serviceClient: sc,
77+
newServiceClient: factory,
7078
}, nil
7179
}
7280

81+
func isAuthError(err error) bool {
82+
if err == nil {
83+
return false
84+
}
85+
if gophercloud.ResponseCodeIs(err, http.StatusUnauthorized) {
86+
return true
87+
}
88+
var errReauth gophercloud.ErrUnableToReauthenticate
89+
if errors.As(err, &errReauth) {
90+
return true
91+
}
92+
var errAfterReauth gophercloud.ErrErrorAfterReauthentication
93+
return errors.As(err, &errAfterReauth)
94+
}
95+
96+
func (c *Client) reconnect() error {
97+
slog.Info("recreating ironic service client after authentication failure")
98+
sc, err := c.newServiceClient()
99+
if err != nil {
100+
slog.Error("failed to recreate ironic service client", "error", err)
101+
return fmt.Errorf("failed to recreate baremetal client: %w", err)
102+
}
103+
if sc.Endpoint == "" {
104+
return fmt.Errorf("recreated baremetal client has no endpoint configured")
105+
}
106+
c.serviceClient = sc
107+
slog.Info("ironic service client reconnected successfully", "endpoint", sc.Endpoint)
108+
return nil
109+
}
110+
73111
// GetNode fetches a node by UUID or name from Ironic.
74112
func (c *Client) GetNode(ctx context.Context, nodeID string) (*nodes.Node, error) {
75113
node, err := nodes.Get(ctx, c.serviceClient, nodeID).Extract()
76114
if err != nil {
115+
if isAuthError(err) {
116+
slog.Info("auth error on GetNode, attempting reconnect", "nodeID", nodeID, "error", err)
117+
if reconnErr := c.reconnect(); reconnErr != nil {
118+
return nil, fmt.Errorf("get node %s: reconnect failed: %w", nodeID, reconnErr)
119+
}
120+
node, err = nodes.Get(ctx, c.serviceClient, nodeID).Extract()
121+
if err != nil {
122+
return nil, fmt.Errorf("get node %s after reconnect: %w", nodeID, err)
123+
}
124+
return node, nil
125+
}
77126
return nil, fmt.Errorf("get node %s: %w", nodeID, err)
78127
}
79128
return node, nil
@@ -93,6 +142,17 @@ func (c *Client) IsNodePowerTransitioning(node *nodes.Node) bool {
93142
func (c *Client) SetPowerState(ctx context.Context, nodeID string, target TargetPowerState) error {
94143
res := nodes.ChangePowerState(ctx, c.serviceClient, nodeID, nodes.PowerStateOpts{Target: target.powerstate})
95144
if err := res.ExtractErr(); err != nil {
145+
if isAuthError(err) {
146+
slog.Info("auth error on SetPowerState, attempting reconnect", "nodeID", nodeID, "target", target.String(), "error", err)
147+
if reconnErr := c.reconnect(); reconnErr != nil {
148+
return fmt.Errorf("failed to set power state on node %s: reconnect failed: %w", nodeID, reconnErr)
149+
}
150+
res = nodes.ChangePowerState(ctx, c.serviceClient, nodeID, nodes.PowerStateOpts{Target: target.powerstate})
151+
if err := res.ExtractErr(); err != nil {
152+
return fmt.Errorf("failed to set power state on node %s after reconnect: %w", nodeID, err)
153+
}
154+
return nil
155+
}
96156
return fmt.Errorf("failed to set power state on node %s: %w", nodeID, err)
97157
}
98158
return nil
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
Copyright 2026.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package ironic
18+
19+
import (
20+
"fmt"
21+
"net/http"
22+
23+
. "github.com/onsi/ginkgo/v2"
24+
. "github.com/onsi/gomega"
25+
26+
"github.com/gophercloud/gophercloud/v2"
27+
)
28+
29+
const (
30+
oldEndpoint = "http://old:6385/v1/"
31+
newEndpoint = "http://new:6385/v1/"
32+
)
33+
34+
var _ = Describe("isAuthError", func() {
35+
It("should return false for nil error", func() {
36+
Expect(isAuthError(nil)).To(BeFalse())
37+
})
38+
39+
It("should return false for generic errors", func() {
40+
Expect(isAuthError(fmt.Errorf("some random error"))).To(BeFalse())
41+
})
42+
43+
It("should return true for 401 ErrUnexpectedResponseCode", func() {
44+
err := gophercloud.ErrUnexpectedResponseCode{
45+
Actual: http.StatusUnauthorized,
46+
Expected: []int{http.StatusOK},
47+
}
48+
Expect(isAuthError(err)).To(BeTrue())
49+
})
50+
51+
It("should return false for 404 ErrUnexpectedResponseCode", func() {
52+
err := gophercloud.ErrUnexpectedResponseCode{
53+
Actual: http.StatusNotFound,
54+
Expected: []int{http.StatusOK},
55+
}
56+
Expect(isAuthError(err)).To(BeFalse())
57+
})
58+
59+
It("should return false for 500 ErrUnexpectedResponseCode", func() {
60+
err := gophercloud.ErrUnexpectedResponseCode{
61+
Actual: http.StatusInternalServerError,
62+
Expected: []int{http.StatusOK},
63+
}
64+
Expect(isAuthError(err)).To(BeFalse())
65+
})
66+
67+
It("should return true for ErrUnableToReauthenticate", func() {
68+
err := gophercloud.ErrUnableToReauthenticate{
69+
ErrOriginal: fmt.Errorf("original"),
70+
ErrReauth: fmt.Errorf("reauth failed"),
71+
}
72+
Expect(isAuthError(err)).To(BeTrue())
73+
})
74+
75+
It("should return true for ErrErrorAfterReauthentication", func() {
76+
err := gophercloud.ErrErrorAfterReauthentication{
77+
ErrOriginal: fmt.Errorf("still failing"),
78+
}
79+
Expect(isAuthError(err)).To(BeTrue())
80+
})
81+
82+
It("should return true for wrapped 401 error", func() {
83+
inner := gophercloud.ErrUnexpectedResponseCode{
84+
Actual: http.StatusUnauthorized,
85+
Expected: []int{http.StatusOK},
86+
}
87+
wrapped := fmt.Errorf("operation failed: %w", inner)
88+
Expect(isAuthError(wrapped)).To(BeTrue())
89+
})
90+
91+
It("should return false for wrapped non-auth error", func() {
92+
inner := gophercloud.ErrUnexpectedResponseCode{
93+
Actual: http.StatusNotFound,
94+
Expected: []int{http.StatusOK},
95+
}
96+
wrapped := fmt.Errorf("operation failed: %w", inner)
97+
Expect(isAuthError(wrapped)).To(BeFalse())
98+
})
99+
})
100+
101+
var _ = Describe("reconnect", func() {
102+
It("should swap the service client on success", func() {
103+
oldSC := &gophercloud.ServiceClient{Endpoint: oldEndpoint}
104+
newSC := &gophercloud.ServiceClient{Endpoint: newEndpoint}
105+
106+
c := &Client{
107+
serviceClient: oldSC,
108+
newServiceClient: func() (*gophercloud.ServiceClient, error) {
109+
return newSC, nil
110+
},
111+
}
112+
113+
Expect(c.reconnect()).To(Succeed())
114+
Expect(c.serviceClient).To(Equal(newSC))
115+
Expect(c.GetEndpoint()).To(Equal(newEndpoint))
116+
})
117+
118+
It("should return error when factory fails", func() {
119+
oldSC := &gophercloud.ServiceClient{Endpoint: oldEndpoint}
120+
121+
c := &Client{
122+
serviceClient: oldSC,
123+
newServiceClient: func() (*gophercloud.ServiceClient, error) {
124+
return nil, fmt.Errorf("keystone is down")
125+
},
126+
}
127+
128+
err := c.reconnect()
129+
Expect(err).To(HaveOccurred())
130+
Expect(err.Error()).To(ContainSubstring("keystone is down"))
131+
Expect(c.serviceClient).To(Equal(oldSC), "should keep old client on failure")
132+
})
133+
134+
It("should return error when new client has empty endpoint", func() {
135+
oldSC := &gophercloud.ServiceClient{Endpoint: oldEndpoint}
136+
137+
c := &Client{
138+
serviceClient: oldSC,
139+
newServiceClient: func() (*gophercloud.ServiceClient, error) {
140+
return &gophercloud.ServiceClient{Endpoint: ""}, nil
141+
},
142+
}
143+
144+
err := c.reconnect()
145+
Expect(err).To(HaveOccurred())
146+
Expect(err.Error()).To(ContainSubstring("no endpoint configured"))
147+
})
148+
})

0 commit comments

Comments
 (0)