all: Add retry logic to CLI end_devices create

Author: happyRip

cmd/ttn-lw-cli/commands/end_devices.go

@@ -1,4 +1,4 @@
-// Copyright © 2020 The Things Network Foundation, The Things Industries B.V.
+// Copyright © 2025 The Things Network Foundation, The Things Industries B.V.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -584,14 +584,18 @@ var (
 				return err
 			}
 
-			application, err := ttnpb.NewApplicationRegistryClient(is).Get(ctx, &ttnpb.GetApplicationRequest{
-				ApplicationIds: devID.ApplicationIds,
-				FieldMask: ttnpb.FieldMask(
-					"network_server_address",
-					"application_server_address",
-					"join_server_address",
-				),
-			})
+			application, err := api.CallWithBackoff(
+				func() (*ttnpb.Application, error) {
+					return ttnpb.NewApplicationRegistryClient(is).Get(ctx, &ttnpb.GetApplicationRequest{
+						ApplicationIds: devID.ApplicationIds,
+						FieldMask: ttnpb.FieldMask(
+							"network_server_address",
+							"application_server_address",
+							"join_server_address",
+						),
+					})
+				},
+			)
 			if err != nil {
 				return err
 			}
@@ -604,7 +608,11 @@ var (
 					return errEndDeviceClaimGeneratedEUI.New()
 				}
 				logger.Debug("request-dev-eui flag set, requesting a DevEUI")
-				devEUIResponse, err := ttnpb.NewApplicationRegistryClient(is).IssueDevEUI(ctx, devID.ApplicationIds)
+				devEUIResponse, err := api.CallWithBackoff(
+					func() (*ttnpb.IssueDevEUIResponse, error) {
+						return ttnpb.NewApplicationRegistryClient(is).IssueDevEUI(ctx, devID.ApplicationIds)
+					},
+				)
 				if err != nil {
 					return err
 				}
@@ -634,26 +642,34 @@ var (
 				if err != nil {
 					return err
 				}
-				claimInfoResp, err := ttnpb.NewEndDeviceClaimingServerClient(dcs).GetInfoByJoinEUI(ctx, &ttnpb.GetInfoByJoinEUIRequest{
-					JoinEui: device.Ids.JoinEui,
-				})
+				claimInfoResp, err := api.CallWithBackoff(
+					func() (*ttnpb.GetInfoByJoinEUIResponse, error) {
+						return ttnpb.NewEndDeviceClaimingServerClient(dcs).GetInfoByJoinEUI(ctx, &ttnpb.GetInfoByJoinEUIRequest{
+							JoinEui: device.Ids.JoinEui,
+						})
+					},
+				)
 				if err != nil {
 					return errEndDeviceClaimInfo.WithCause(err)
 				}
 				if !claimInfoResp.SupportsClaiming {
 					return errClaimingNotSupported.WithAttributes("join_eui", types.MustEUI64(device.Ids.JoinEui).String())
 				}
-				_, err = ttnpb.NewEndDeviceClaimingServerClient(dcs).Claim(ctx, &ttnpb.ClaimEndDeviceRequest{
-					TargetApplicationIds: device.Ids.ApplicationIds,
-					TargetDeviceId:       device.Ids.DeviceId,
-					SourceDevice: &ttnpb.ClaimEndDeviceRequest_AuthenticatedIdentifiers_{
-						AuthenticatedIdentifiers: &ttnpb.ClaimEndDeviceRequest_AuthenticatedIdentifiers{
-							JoinEui:            device.Ids.JoinEui,
-							DevEui:             device.Ids.DevEui,
-							AuthenticationCode: device.ClaimAuthenticationCode.Value,
-						},
+				_, err = api.CallWithBackoff(
+					func() (*ttnpb.EndDeviceIdentifiers, error) {
+						return ttnpb.NewEndDeviceClaimingServerClient(dcs).Claim(ctx, &ttnpb.ClaimEndDeviceRequest{
+							TargetApplicationIds: device.Ids.ApplicationIds,
+							TargetDeviceId:       device.Ids.DeviceId,
+							SourceDevice: &ttnpb.ClaimEndDeviceRequest_AuthenticatedIdentifiers_{
+								AuthenticatedIdentifiers: &ttnpb.ClaimEndDeviceRequest_AuthenticatedIdentifiers{
+									JoinEui:            device.Ids.JoinEui,
+									DevEui:             device.Ids.DevEui,
+									AuthenticationCode: device.ClaimAuthenticationCode.Value,
+								},
+							},
+						})
 					},
-				})
+				)
 				if err != nil {
 					return errEndDeviceClaim.WithCause(err)
 				}
@@ -672,9 +688,13 @@ var (
 			if err := isDevice.SetFields(device, append(isPaths, "ids")...); err != nil {
 				return err
 			}
-			isRes, err := ttnpb.NewEndDeviceRegistryClient(is).Create(ctx, &ttnpb.CreateEndDeviceRequest{
-				EndDevice: isDevice,
-			})
+			isRes, err := api.CallWithBackoff(
+				func() (*ttnpb.EndDevice, error) {
+					return ttnpb.NewEndDeviceRegistryClient(is).Create(ctx, &ttnpb.CreateEndDeviceRequest{
+						EndDevice: isDevice,
+					})
+				},
+			)
 			if err != nil {
 				return err
 			}

cmd/ttn-lw-cli/internal/api/grpc.go

@@ -28,8 +28,11 @@ import (
 	"go.thethings.network/lorawan-stack/v3/pkg/rpcmiddleware/rpclog"
 	"go.thethings.network/lorawan-stack/v3/pkg/rpcmiddleware/rpcretry"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 var (
@@ -188,3 +191,60 @@ func CloseAll() {
 		conn.Close()
 	}
 }
+
+type retry[T proto.Message] struct {
+	backoff    time.Duration
+	maxRetries int
+}
+
+type retryOption[T proto.Message] func(*retry[T])
+
+// WithBackoff sets the initial backoff duration between retries.
+func WithBackoff[T proto.Message](d time.Duration) retryOption[T] {
+	return func(r *retry[T]) {
+		r.backoff = d
+	}
+}
+
+// WithMaxRetries sets the maximum number of retry attempts.
+func WithMaxRetries[T proto.Message](n int) retryOption[T] {
+	return func(r *retry[T]) {
+		r.maxRetries = n
+	}
+}
+
+// CallWithBackoff executes a gRPC call with retry logic and exponential backoff.
+//
+// It takes a function `call` that performs the actual gRPC request and retries it
+// if the error returned is a gRPC `codes.ResourceExhausted` error. The retry behavior
+// (initial backoff duration and maximum number of retries) can be customized by
+// providing functional options (e.g., WithBackoff, WithMaxRetries). If no options are
+// provided, it defaults to a 1-second backoff and 5 maximum retries.
+func CallWithBackoff[T proto.Message](call func() (T, error), opts ...retryOption[T]) (T, error) {
+	r := retry[T]{
+		backoff:    time.Second,
+		maxRetries: 5,
+	}
+	for _, opt := range opts {
+		opt(&r)
+	}
+	return r.callWithBackoff(call)
+}
+
+func (r retry[T]) callWithBackoff(call func() (T, error)) (T, error) {
+	var zero T
+	backoff := r.backoff
+	for attempt := 0; attempt < r.maxRetries; attempt++ {
+		v, err := call()
+		if err == nil {
+			return v, nil
+		}
+		st, ok := status.FromError(err)
+		if !ok || st.Code() != codes.ResourceExhausted {
+			return zero, err
+		}
+		time.Sleep(backoff)
+		backoff *= 2
+	}
+	return call()
+}