diff --git a/cmd/boulder-wfe2/main.go b/cmd/boulder-wfe2/main.go index 93d5b987a42..7da4e516201 100644 --- a/cmd/boulder-wfe2/main.go +++ b/cmd/boulder-wfe2/main.go @@ -19,6 +19,7 @@ import ( "github.com/letsencrypt/boulder/goodkey/sagoodkey" bgrpc "github.com/letsencrypt/boulder/grpc" "github.com/letsencrypt/boulder/grpc/noncebalancer" + "github.com/letsencrypt/boulder/grpc/noncebalancerv2" "github.com/letsencrypt/boulder/issuance" "github.com/letsencrypt/boulder/nonce" rapb "github.com/letsencrypt/boulder/ra/proto" @@ -318,9 +319,11 @@ func main() { cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to get nonce service") gnc := nonce.NewGetter(getNonceConn) - if c.WFE.RedeemNonceService.SRVResolver != noncebalancer.SRVResolverScheme { + if c.WFE.RedeemNonceService.SRVResolver != noncebalancer.SRVResolverScheme && + c.WFE.RedeemNonceService.SRVResolver != noncebalancerv2.SRVResolverScheme { cmd.Fail(fmt.Sprintf( - "'redeemNonceService.SRVResolver' must be set to %q", noncebalancer.SRVResolverScheme), + "'redeemNonceService.SRVResolver' must be set to %q or %q", + noncebalancer.SRVResolverScheme, noncebalancerv2.SRVResolverScheme), ) } redeemNonceConn, err := bgrpc.ClientSetup(c.WFE.RedeemNonceService, tlsConfig, stats, clk) diff --git a/cmd/config.go b/cmd/config.go index 414214e282a..acbfb4aff9c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -306,7 +306,7 @@ type GRPCClientConfig struct { // implementation of the SRV resolver should be used. The default is 'srv' // For more details, see the documentation in: // grpc/internal/resolver/dns/dns_resolver.go. - SRVResolver string `validate:"excluded_with=ServerAddress,isdefault|oneof=srv nonce-srv"` + SRVResolver string `validate:"excluded_with=ServerAddress,isdefault|oneof=srv nonce-srv nonce-srv-v2"` // ServerAddress is a single : or `:` that // the gRPC client will, if necessary, resolve via DNS and then connect to. diff --git a/grpc/internal/resolver/dns/dns_resolver.go b/grpc/internal/resolver/dns/dns_resolver.go index a25bee078df..606986049bb 100644 --- a/grpc/internal/resolver/dns/dns_resolver.go +++ b/grpc/internal/resolver/dns/dns_resolver.go @@ -40,6 +40,7 @@ import ( "github.com/letsencrypt/boulder/bdns" "github.com/letsencrypt/boulder/grpc/internal/backoff" "github.com/letsencrypt/boulder/grpc/noncebalancer" + "github.com/letsencrypt/boulder/grpc/noncebalancerv2" ) var logger = grpclog.Component("srv") @@ -54,6 +55,7 @@ var ( func init() { resolver.Register(NewDefaultSRVBuilder()) resolver.Register(NewNonceSRVBuilder()) + resolver.Register(NewNonceSRVBuilderV2()) } const defaultDNSSvrPort = "53" @@ -96,6 +98,12 @@ func NewNonceSRVBuilder() resolver.Builder { return &srvBuilder{scheme: noncebalancer.SRVResolverScheme, balancer: noncebalancer.Name} } +// NewNonceSRVBuilderV2 creates a srvBuilder which is used to factory SRV DNS +// resolvers with the v2 nonce balancer used by nonce-service clients. +func NewNonceSRVBuilderV2() resolver.Builder { + return &srvBuilder{scheme: noncebalancerv2.SRVResolverScheme, balancer: noncebalancerv2.Name} +} + type srvBuilder struct { scheme string balancer string diff --git a/grpc/noncebalancerv2/balancer.go b/grpc/noncebalancerv2/balancer.go new file mode 100644 index 00000000000..3350bcb1022 --- /dev/null +++ b/grpc/noncebalancerv2/balancer.go @@ -0,0 +1,213 @@ +package noncebalancerv2 + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +var logger = grpclog.Component("noncebalancerv2") + +// nonceBalancer implements balancer.Balancer. It is a near-exact copy of +// grpc/balancer/base/balancer.go's baseBalancer with one difference: +// regeneratePicker passes ALL resolver-tracked SubConns to the picker, not just +// READY ones. This allows the picker to distinguish "backend is temporarily +// reconnecting" (queue the RPC via ErrNoSubConnAvailable) from "prefix is +// genuinely unknown" (fail with ErrNoBackendsMatchPrefix). +type nonceBalancer struct { + cc balancer.ClientConn + + csEvltr *balancer.ConnectivityStateEvaluator + state connectivity.State + + subConns *resolver.AddressMapV2[balancer.SubConn] + scStates map[balancer.SubConn]connectivity.State + picker balancer.Picker + config base.Config + + resolverErr error // the last error reported by the resolver; cleared on successful resolution + connErr error // the last connection error; cleared upon leaving TransientFailure +} + +func (b *nonceBalancer) ResolverError(err error) { + b.resolverErr = err + if b.subConns.Len() == 0 { + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.state, + Picker: b.picker, + }) +} + +func (b *nonceBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + // TODO: handle s.ResolverState.ServiceConfig? + if logger.V(2) { + logger.Info("noncebalancer: got new ClientConn state: ", s) + } + // Successful resolution; clear resolver error and ensure we return nil. + b.resolverErr = nil + // addrsSet is the set converted from addrs, it's used for quick lookup of an address. + addrsSet := resolver.NewAddressMapV2[any]() + for _, a := range s.ResolverState.Addresses { + addrsSet.Set(a, nil) + if _, ok := b.subConns.Get(a); !ok { + // a is a new address (not existing in b.subConns). + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + HealthCheckEnabled: b.config.HealthCheck, + StateListener: func(scs balancer.SubConnState) { b.updateSubConnState(sc, scs) }, + } + sc, err := b.cc.NewSubConn([]resolver.Address{a}, opts) + if err != nil { + logger.Warningf("noncebalancer: failed to create new SubConn: %v", err) + continue + } + b.subConns.Set(a, sc) + b.scStates[sc] = connectivity.Idle + b.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Idle) + sc.Connect() + } + } + for _, a := range b.subConns.Keys() { + sc, _ := b.subConns.Get(a) + // a was removed by resolver. + if _, ok := addrsSet.Get(a); !ok { + sc.Shutdown() + b.subConns.Delete(a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in updateSubConnState. + } + } + // If resolver state contains no addresses, return an error so ClientConn + // will trigger re-resolve. Also records this as a resolver error, so when + // the overall state turns transient failure, the error message will have + // the zero address information. + if len(s.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + return nil +} + +// mergeErrors builds an error from the last connection error and the last +// resolver error. Must only be called if b.state is TransientFailure. +func (b *nonceBalancer) mergeErrors() error { + // connErr must always be non-nil unless there are no SubConns, in which + // case resolverErr must be non-nil. + if b.connErr == nil { + return fmt.Errorf("last resolver error: %v", b.resolverErr) + } + if b.resolverErr == nil { + return fmt.Errorf("last connection error: %v", b.connErr) + } + return fmt.Errorf("last connection error: %v; last resolver error: %v", b.connErr, b.resolverErr) +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker +// from it. The picker is +// - errPicker if the balancer is in TransientFailure, +// - a nonce picker with all READY SubConns and all known SubConns otherwise. +// +// This is the only method that differs from baseBalancer: it builds both a +// READY set and a not-READY set from b.subConns. baseBalancer only builds the +// READY set. +func (b *nonceBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = base.NewErrPicker(b.mergeErrors()) + return + } + readySCs := make(map[balancer.SubConn]resolver.Address) + notReadySCs := make(map[balancer.SubConn]resolver.Address) + + for _, addr := range b.subConns.Keys() { + sc, _ := b.subConns.Get(addr) + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[sc] = addr + } else { + notReadySCs[sc] = addr + } + } + b.picker = &picker{ + readyBackends: readySCs, + notReadyBackends: notReadySCs, + } +} + +// UpdateSubConnState is a nop because a StateListener is always set in NewSubConn. +func (b *nonceBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + logger.Errorf("noncebalancer: UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *nonceBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + s := state.ConnectivityState + if logger.V(2) { + logger.Infof("noncebalancer: handle SubConn state change: %p, %v", sc, s) + } + oldS, ok := b.scStates[sc] + if !ok { + if logger.V(2) { + logger.Infof("noncebalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + } + return + } + if oldS == connectivity.TransientFailure && + (s == connectivity.Connecting || s == connectivity.Idle) { + // Once a subconn enters TRANSIENT_FAILURE, ignore subsequent IDLE or + // CONNECTING transitions to prevent the aggregated state from being + // always CONNECTING when many backends exist but are all down. + if s == connectivity.Idle { + sc.Connect() + } + return + } + b.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + case connectivity.TransientFailure: + // Save error to be reported via picker. + b.connErr = state.ConnectionError + } + + b.state = b.csEvltr.RecordTransition(oldS, s) + + // Regenerate picker when one of the following happens: + // - this sc entered or left ready + // - the aggregated state of balancer is TransientFailure + // (may need to update error message) + if (s == connectivity.Ready) != (oldS == connectivity.Ready) || + b.state == connectivity.TransientFailure { + b.regeneratePicker() + } + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) +} + +// Close is a nop because base balancer doesn't have internal state to clean up, +// and it doesn't need to call Shutdown for the SubConns. +func (b *nonceBalancer) Close() { +} + +// ExitIdle is a nop because the base balancer attempts to stay connected to +// all SubConns at all times. +func (b *nonceBalancer) ExitIdle() { +} diff --git a/grpc/noncebalancerv2/noncebalancer.go b/grpc/noncebalancerv2/noncebalancer.go new file mode 100644 index 00000000000..eb142e2015c --- /dev/null +++ b/grpc/noncebalancerv2/noncebalancer.go @@ -0,0 +1,59 @@ +package noncebalancerv2 + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +const ( + // Name is the name used to register the nonce balancer with the gRPC + // runtime. + Name = "noncev2" + + // SRVResolverScheme is the scheme used to invoke an instance of the SRV + // resolver which will use the noncebalancer to pick backends. It would be + // ideal to export this from the SRV resolver package but that package is + // internal. + SRVResolverScheme = "nonce-srv-v2" +) + +type builder struct { + name string + config base.Config +} + +// NewBalancerBuilder returns a nonce balancer builder configured by the +// provided config. +func NewBalancerBuilder(name string, config base.Config) balancer.Builder { + return &builder{ + name: name, + config: config, + } +} + +func (bb *builder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + bal := &nonceBalancer{ + cc: cc, + + subConns: resolver.NewAddressMapV2[balancer.SubConn](), + scStates: make(map[balancer.SubConn]connectivity.State), + csEvltr: &balancer.ConnectivityStateEvaluator{}, + config: bb.config, + state: connectivity.Connecting, + } + // Initialize picker to a picker that always returns + // ErrNoSubConnAvailable, because when state of a SubConn changes, we + // may call UpdateState with this picker. + bal.picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable) + return bal +} + +func (bb *builder) Name() string { + return bb.name +} + +func init() { + balancer.Register(NewBalancerBuilder(Name, base.Config{})) +} diff --git a/grpc/noncebalancerv2/picker.go b/grpc/noncebalancerv2/picker.go new file mode 100644 index 00000000000..900fda5ec9b --- /dev/null +++ b/grpc/noncebalancerv2/picker.go @@ -0,0 +1,103 @@ +package noncebalancerv2 + +import ( + "errors" + "sync" + + "github.com/letsencrypt/boulder/nonce" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" +) + +// ErrNoBackendsMatchPrefix indicates that no backends were found which match +// the nonce prefix provided in the RPC context. This can happen when the +// provided nonce is stale, valid but the backend has since been removed from +// the balancer, or valid but the backend has not yet been added to the +// balancer. +// +// In any case, when the WFE receives this error it will return a badNonce error +// to the ACME client. Note that the WFE uses exact pointer comparison to +// detect that the status it receives is this exact status object, so don't +// wrap this with fmt.Errorf when returning it. +var ErrNoBackendsMatchPrefix = status.New(codes.Unavailable, "no backends match the nonce prefix") +var errMissingPrefixCtxKey = errors.New("nonce.PrefixCtxKey value required in RPC context") +var errMissingHMACKeyCtxKey = errors.New("nonce.HMACKeyCtxKey value required in RPC context") +var errInvalidPrefixCtxKeyType = errors.New("nonce.PrefixCtxKey value in RPC context must be a string") +var errInvalidHMACKeyCtxKeyType = errors.New("nonce.HMACKeyCtxKey value in RPC context must be a byte slice") + +// picker implements the balancer.Picker interface. It picks a backend (SubConn) +// based on the nonce prefix contained in each request's Context. +type picker struct { + // readyBackends contains only SubConns in the READY state. + readyBackends map[balancer.SubConn]resolver.Address + + // notReadyBackends contains SubConns that the resolver reports but are not + // in the READY state. + notReadyBackends map[balancer.SubConn]resolver.Address + + prefixToReady map[string]balancer.SubConn + prefixToNotReady map[string]balancer.SubConn + prefixToBackendOnce sync.Once +} + +// Pick is called by the gRPC runtime for each RPC. It routes the RPC to the +// backend matching the nonce prefix in the request context. If the backend +// exists but is not READY, it returns ErrNoSubConnAvailable to tell gRPC to +// queue the RPC until a new picker is available (see picker_wrapper.go:159). +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Get the HMAC key from the RPC context. + hmacKeyVal := info.Ctx.Value(nonce.HMACKeyCtxKey{}) + if hmacKeyVal == nil { + // This should never happen. + return balancer.PickResult{}, errMissingHMACKeyCtxKey + } + hmacKey, ok := hmacKeyVal.([]byte) + if !ok { + // This should never happen. + return balancer.PickResult{}, errInvalidHMACKeyCtxKeyType + } + + p.prefixToBackendOnce.Do(func() { + // First call to Pick with a new Picker. + p.prefixToReady = make(map[string]balancer.SubConn) + for sc, addr := range p.readyBackends { + p.prefixToReady[nonce.DerivePrefix(addr.Addr, hmacKey)] = sc + } + p.prefixToNotReady = make(map[string]balancer.SubConn) + for sc, addr := range p.notReadyBackends { + p.prefixToNotReady[nonce.DerivePrefix(addr.Addr, hmacKey)] = sc + } + }) + + destPrefixVal := info.Ctx.Value(nonce.PrefixCtxKey{}) + if destPrefixVal == nil { + // This should never happen. + return balancer.PickResult{}, errMissingPrefixCtxKey + } + destPrefix, ok := destPrefixVal.(string) + if !ok { + // This should never happen. + return balancer.PickResult{}, errInvalidPrefixCtxKeyType + } + + sc, ok := p.prefixToReady[destPrefix] + if ok { + // The backend exists and is READY. + return balancer.PickResult{SubConn: sc}, nil + } + + _, ok = p.prefixToNotReady[destPrefix] + if ok { + // The backend exists but is not READY (e.g. reconnecting after a + // GOAWAY). Tell gRPC to wait for a new picker, which will be provided + // when the SubConn's state changes. + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } + + // The backend doesn't exist at all: stale nonce, or backend removed from + // balancer. Return a non-retryable error so the WFE can return return a + // badNonce error. + return balancer.PickResult{}, ErrNoBackendsMatchPrefix.Err() +} diff --git a/grpc/noncebalancerv2/picker_test.go b/grpc/noncebalancerv2/picker_test.go new file mode 100644 index 00000000000..fbc2b939d88 --- /dev/null +++ b/grpc/noncebalancerv2/picker_test.go @@ -0,0 +1,170 @@ +package noncebalancerv2 + +import ( + "context" + "testing" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" + + "github.com/letsencrypt/boulder/nonce" + "github.com/letsencrypt/boulder/test" +) + +func TestPickerPicksCorrectBackend(t *testing.T) { + sc, p := setupTestPicker(t) + + hmacKey := []byte("Kala namak") + prefix := nonce.DerivePrefix(sc.addrs[0].Addr, hmacKey) + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, prefix) + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, hmacKey) + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertNotError(t, err, "Pick failed") + test.AssertDeepEquals(t, sc, gotPick.SubConn) +} + +func TestPickerMissingPrefixInCtx(t *testing.T) { + _, p := setupTestPicker(t) + + testCtx := context.WithValue(context.Background(), nonce.HMACKeyCtxKey{}, []byte("Kala namak")) + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, errMissingPrefixCtxKey) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerInvalidPrefixInCtx(t *testing.T) { + _, p := setupTestPicker(t) + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, 9) + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, []byte("foobar")) + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, errInvalidPrefixCtxKeyType) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerMissingHMACKeyInCtx(t *testing.T) { + _, p := setupTestPicker(t) + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, "HNmOnt8w") + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, errMissingHMACKeyCtxKey) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerInvalidHMACKeyInCtx(t *testing.T) { + _, p := setupTestPicker(t) + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, "HNmOnt8w") + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, 9) + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, errInvalidHMACKeyCtxKeyType) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerUnknownPrefix(t *testing.T) { + _, p := setupTestPicker(t) + + hmacKey := []byte("Kala namak") + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, "rUsTrUin") + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, hmacKey) + info := balancer.PickInfo{Ctx: testCtx} + + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, ErrNoBackendsMatchPrefix.Err()) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerSubConnReconnecting(t *testing.T) { + sc := &subConn{} + addr := resolver.Address{Addr: "10.77.77.77:8080"} + sc.UpdateAddresses([]resolver.Address{addr}) + + hmacKey := []byte("Kala namak") + prefix := nonce.DerivePrefix(addr.Addr, hmacKey) + + // Build a picker where the SubConn is known but not READY. + p := &picker{ + readyBackends: map[balancer.SubConn]resolver.Address{}, + notReadyBackends: map[balancer.SubConn]resolver.Address{sc: addr}, + } + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, prefix) + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, hmacKey) + info := balancer.PickInfo{Ctx: testCtx} + + // Should return ErrNoSubConnAvailable (queue the RPC) not + // ErrNoBackendsMatchPrefix (fail the RPC). + gotPick, err := p.Pick(info) + test.AssertErrorIs(t, err, balancer.ErrNoSubConnAvailable) + test.AssertNil(t, gotPick.SubConn, "subConn should be nil") +} + +func TestPickerSubConnBecomesReady(t *testing.T) { + sc := &subConn{} + addr := resolver.Address{Addr: "10.77.77.77:8080"} + sc.UpdateAddresses([]resolver.Address{addr}) + + hmacKey := []byte("Kala namak") + prefix := nonce.DerivePrefix(addr.Addr, hmacKey) + + // First picker: SubConn is not READY. + p1 := &picker{ + readyBackends: map[balancer.SubConn]resolver.Address{}, + notReadyBackends: map[balancer.SubConn]resolver.Address{sc: addr}, + } + + testCtx := context.WithValue(context.Background(), nonce.PrefixCtxKey{}, prefix) + testCtx = context.WithValue(testCtx, nonce.HMACKeyCtxKey{}, hmacKey) + info := balancer.PickInfo{Ctx: testCtx} + + _, err := p1.Pick(info) + test.AssertErrorIs(t, err, balancer.ErrNoSubConnAvailable) + + // Second picker: SubConn is now READY (simulates picker rebuild after + // SubConn reconnects). + p2 := &picker{ + readyBackends: map[balancer.SubConn]resolver.Address{sc: addr}, + notReadyBackends: map[balancer.SubConn]resolver.Address{}, + } + + gotPick, err := p2.Pick(info) + test.AssertNotError(t, err, "Pick failed after SubConn became READY") + test.AssertDeepEquals(t, sc, gotPick.SubConn) +} + +// setupTestPicker creates a picker with a single READY SubConn for testing. +func setupTestPicker(t *testing.T) (*subConn, balancer.Picker) { + t.Helper() + + sc := &subConn{} + addr := resolver.Address{Addr: "10.77.77.77:8080"} + sc.UpdateAddresses([]resolver.Address{addr}) + + p := &picker{ + readyBackends: map[balancer.SubConn]resolver.Address{sc: addr}, + notReadyBackends: map[balancer.SubConn]resolver.Address{}, + } + return sc, p +} + +// subConn is a test mock which implements the balancer.SubConn interface. +type subConn struct { + balancer.SubConn + addrs []resolver.Address +} + +func (s *subConn) UpdateAddresses(addrs []resolver.Address) { + s.addrs = addrs +} diff --git a/test/config-next/wfe2.json b/test/config-next/wfe2.json index 6c14df15a83..0009838809c 100644 --- a/test/config-next/wfe2.json +++ b/test/config-next/wfe2.json @@ -74,7 +74,7 @@ "domain": "service.consul" } ], - "srvResolver": "nonce-srv", + "srvResolver": "nonce-srv-v2", "timeout": "15s", "noWaitForReady": true, "hostOverride": "nonce.boulder" diff --git a/wfe2/verify.go b/wfe2/verify.go index c7431d1c6c4..2ea50eeb14a 100644 --- a/wfe2/verify.go +++ b/wfe2/verify.go @@ -24,6 +24,7 @@ import ( "github.com/letsencrypt/boulder/goodkey" "github.com/letsencrypt/boulder/grpc" nb "github.com/letsencrypt/boulder/grpc/noncebalancer" + nbv2 "github.com/letsencrypt/boulder/grpc/noncebalancerv2" "github.com/letsencrypt/boulder/nonce" noncepb "github.com/letsencrypt/boulder/nonce/proto" sapb "github.com/letsencrypt/boulder/sa/proto" @@ -228,7 +229,7 @@ func (wfe *WebFrontEndImpl) validNonce(ctx context.Context, header jose.Header) resp, err := wfe.rnc.Redeem(ctx, &noncepb.NonceMessage{Nonce: header.Nonce}) if err != nil { rpcStatus, ok := status.FromError(err) - if ok && rpcStatus == nb.ErrNoBackendsMatchPrefix { + if ok && (rpcStatus == nb.ErrNoBackendsMatchPrefix || rpcStatus == nbv2.ErrNoBackendsMatchPrefix) { // Getting our sentinel ErrNoBackendsMatchPrefix status.Status means that // the nonce backend which issued this nonce is presently unreachable or // unrecognized by this WFE. As this is a transient failure, the client diff --git a/wfe2/wfe.go b/wfe2/wfe.go index 7c12ad7e608..3e23c3a48c1 100644 --- a/wfe2/wfe.go +++ b/wfe2/wfe.go @@ -32,7 +32,8 @@ import ( "github.com/letsencrypt/boulder/features" "github.com/letsencrypt/boulder/goodkey" bgrpc "github.com/letsencrypt/boulder/grpc" - _ "github.com/letsencrypt/boulder/grpc/noncebalancer" // imported for its init function. + _ "github.com/letsencrypt/boulder/grpc/noncebalancer" // imported for its init function. + _ "github.com/letsencrypt/boulder/grpc/noncebalancerv2" // imported for its init function. "github.com/letsencrypt/boulder/identifier" "github.com/letsencrypt/boulder/issuance" blog "github.com/letsencrypt/boulder/log"