diff --git a/pkg/cfn/manager/api.go b/pkg/cfn/manager/api.go index f00b132ff0..f81208c45e 100644 --- a/pkg/cfn/manager/api.go +++ b/pkg/cfn/manager/api.go @@ -544,7 +544,6 @@ func nonTransitionalReadyStackStatuses() []types.StackStatus { return []types.StackStatus{ types.StackStatusCreateComplete, types.StackStatusUpdateComplete, - types.StackStatusRollbackComplete, types.StackStatusUpdateRollbackComplete, } } diff --git a/pkg/ctl/cmdutils/filter/nodegroup_filter.go b/pkg/ctl/cmdutils/filter/nodegroup_filter.go index 94e4ef39fc..07eabbee34 100644 --- a/pkg/ctl/cmdutils/filter/nodegroup_filter.go +++ b/pkg/ctl/cmdutils/filter/nodegroup_filter.go @@ -2,8 +2,10 @@ package filter import ( "context" + "fmt" "strings" + cfntypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/kris-nova/logger" "k8s.io/apimachinery/pkg/util/sets" @@ -126,6 +128,22 @@ func (f *NodeGroupFilter) loadLocalAndRemoteNodegroups(ctx context.Context, eksA if err != nil { return err } + + if f.onlyLocal { + localNames := sets.New(clusterConfig.GetAllNodeGroupNames()...) + var rolledBack []string + for _, s := range nodeGroupsWithStacks { + if s.Stack != nil && s.Stack.StackStatus == cfntypes.StackStatusRollbackComplete && localNames.Has(s.NodeGroupName) { + rolledBack = append(rolledBack, s.NodeGroupName) + } + } + if len(rolledBack) > 0 { + return fmt.Errorf("nodegroup(s) %q have a CloudFormation stack in ROLLBACK_COMPLETE state; "+ + "delete the failed stack(s) with 'eksctl delete nodegroup --cluster=%s --name=' and then retry creation", + strings.Join(rolledBack, ", "), clusterConfig.Metadata.Name) + } + } + for _, s := range nodeGroupsWithStacks { f.remoteNodegroups.Insert(s.NodeGroupName) } diff --git a/pkg/ctl/cmdutils/filter/nodegroup_filter_test.go b/pkg/ctl/cmdutils/filter/nodegroup_filter_test.go index b7e99a7997..df938e0b62 100644 --- a/pkg/ctl/cmdutils/filter/nodegroup_filter_test.go +++ b/pkg/ctl/cmdutils/filter/nodegroup_filter_test.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/aws" + cfntypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/stretchr/testify/mock" @@ -140,6 +141,81 @@ var _ = Describe("nodegroup filter", func() { }) }) + Context("ROLLBACK_COMPLETE handling", func() { + var ( + filter *NodeGroupFilter + cfg *api.ClusterConfig + mockProvider *mockprovider.MockProvider + ) + + BeforeEach(func() { + cfg = newClusterConfig() + addGroupA(cfg) + + filter = NewNodeGroupFilter() + + mockProvider = mockprovider.NewMockProvider() + mockProvider.MockEKS().On("ListNodegroups", mock.Anything, mock.Anything, mock.Anything).Return(&eks.ListNodegroupsOutput{Nodegroups: nil}, nil) + }) + + It("should return an error when SetOnlyLocal finds a config nodegroup in ROLLBACK_COMPLETE", func() { + mockLister := newMockStackListerWithStacks([]manager.NodeGroupStack{ + { + NodeGroupName: "test-ng1a", + Stack: &manager.Stack{ + StackStatus: cfntypes.StackStatusRollbackComplete, + }, + }, + { + NodeGroupName: "test-ng2a", + Stack: &manager.Stack{ + StackStatus: cfntypes.StackStatusCreateComplete, + }, + }, + }) + + err := filter.SetOnlyLocal(context.Background(), mockProvider.EKS(), mockLister, cfg) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ROLLBACK_COMPLETE")) + Expect(err.Error()).To(ContainSubstring("test-ng1a")) + Expect(err.Error()).To(ContainSubstring("eksctl delete nodegroup")) + }) + + It("should not error when a non-config nodegroup is in ROLLBACK_COMPLETE", func() { + mockLister := newMockStackListerWithStacks([]manager.NodeGroupStack{ + { + NodeGroupName: "unrelated-ng", + Stack: &manager.Stack{ + StackStatus: cfntypes.StackStatusRollbackComplete, + }, + }, + { + NodeGroupName: "test-ng1a", + Stack: &manager.Stack{ + StackStatus: cfntypes.StackStatusCreateComplete, + }, + }, + }) + + err := filter.SetOnlyLocal(context.Background(), mockProvider.EKS(), mockLister, cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should not error when SetOnlyRemote finds a config nodegroup in ROLLBACK_COMPLETE", func() { + mockLister := newMockStackListerWithStacks([]manager.NodeGroupStack{ + { + NodeGroupName: "test-ng1a", + Stack: &manager.Stack{ + StackStatus: cfntypes.StackStatusRollbackComplete, + }, + }, + }) + + err := filter.SetOnlyRemote(context.Background(), mockProvider.EKS(), mockLister, cfg) + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("ForEach", func() { It("should iterate over unique nodegroups, apply defaults and validate", func() { @@ -668,7 +744,7 @@ func (s *mockStackLister) ListNodeGroupStacksWithStatuses(_ context.Context) ([] } func newMockStackLister(ngs ...string) *mockStackLister { - stacks := make([]manager.NodeGroupStack, 0) + stacks := make([]manager.NodeGroupStack, 0, len(ngs)) for _, ng := range ngs { stacks = append(stacks, manager.NodeGroupStack{ NodeGroupName: ng, @@ -678,3 +754,9 @@ func newMockStackLister(ngs ...string) *mockStackLister { nodesResult: stacks, } } + +func newMockStackListerWithStacks(stacks []manager.NodeGroupStack) *mockStackLister { + return &mockStackLister{ + nodesResult: stacks, + } +} diff --git a/pkg/eks/nodegroup_service.go b/pkg/eks/nodegroup_service.go index 6e76030fe5..a7df24d7b6 100644 --- a/pkg/eks/nodegroup_service.go +++ b/pkg/eks/nodegroup_service.go @@ -306,7 +306,7 @@ func ValidateExistingNodeGroupsForCompatibility(ctx context.Context, cfg *api.Cl } if len(incompatibleNodeGroups) == 0 { - logger.Info("all nodegroups have up-to-date cloudformation templates") + logger.Info("all nodegroups have compatible shared security group configuration") return nil }