Skip to content

Commit 8ce058d

Browse files
drewmalindrewmalin
committed
e2e nebius k8s
1 parent 8cb0d49 commit 8ce058d

11 files changed

Lines changed: 723 additions & 393 deletions

File tree

‎internal/validation/suite.go‎

Lines changed: 163 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,15 @@ type NetworkValidationOpts struct {
133133
Location string
134134
CidrBlock string
135135
PublicSubnetCidrBlock string
136+
Tags map[string]string
136137
}
137138

138139
func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValidationOpts) {
139140
if testing.Short() {
140141
t.Skip("Skipping validation tests in short mode")
141142
}
142143

144+
// Set a default timeout of 15 minutes for the validation suite
143145
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
144146
defer cancel()
145147

@@ -148,6 +150,7 @@ func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValid
148150
t.Fatalf("Failed to create client for %s: %v", config.Credential.GetCloudProviderID(), err)
149151
}
150152

153+
// Test #1: ValidateCreateVPC
151154
var vpcID v1.CloudProviderResourceID
152155
t.Run("ValidateCreateVPC", func(t *testing.T) {
153156
vpc, err := v1.ValidateCreateVPC(ctx, client, v1.CreateVPCArgs{
@@ -158,11 +161,13 @@ func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValid
158161
Subnets: []v1.CreateSubnetArgs{
159162
{CidrBlock: opts.PublicSubnetCidrBlock, Type: v1.SubnetTypePublic},
160163
},
164+
Tags: opts.Tags,
161165
})
162166
require.NoError(t, err, "ValidateCreateVPC should pass")
163167
vpcID = vpc.ID
164168
})
165169

170+
// The VPC was created successfully -- create a defer function to delete the VPC if the tests fail
166171
deletionSucceeded := false
167172
defer func() {
168173
if !deletionSucceeded && vpcID != "" {
@@ -176,6 +181,7 @@ func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValid
176181
}
177182
}()
178183

184+
// Test #2: ValidateGetVPC
179185
t.Run("ValidateGetVPC", func(t *testing.T) {
180186
vpc, err := v1.ValidateGetVPC(ctx, client, v1.GetVPCArgs{
181187
ID: vpcID,
@@ -184,19 +190,22 @@ func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValid
184190
require.NotNil(t, vpc)
185191
})
186192

193+
// Test #3: WaitForVPCToBeAvailable
187194
t.Run("WaitForVPCToBeAvailable", func(t *testing.T) {
188-
err := v1.WaitForVPCPredicate(ctx, client, v1.GetVPCArgs{ID: vpcID},
189-
v1.WaitForVPCPredicateOpts{
190-
Predicate: func(vpc *v1.VPC) bool {
191-
return vpc.Status == v1.VPCStatusAvailable
192-
},
193-
Timeout: 5 * time.Minute,
194-
Interval: 5 * time.Second,
195+
err := WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.VPC]{
196+
GetResource: func() (*v1.VPC, error) {
197+
return client.GetVPC(ctx, v1.GetVPCArgs{ID: vpcID})
195198
},
196-
)
199+
Predicate: func(vpc *v1.VPC) bool {
200+
return vpc.Status == v1.VPCStatusAvailable
201+
},
202+
Timeout: 5 * time.Minute,
203+
Interval: 5 * time.Second,
204+
})
197205
require.NoError(t, err, "WaitForVPCToBeAvailable should pass")
198206
})
199207

208+
// Test #4: ValidateDeleteVPC
200209
t.Run("ValidateDeleteVPC", func(t *testing.T) {
201210
err := v1.ValidateDeleteVPC(ctx, client, v1.DeleteVPCArgs{
202211
ID: vpcID,
@@ -205,12 +214,20 @@ func RunNetworkValidation(t *testing.T, config ProviderConfig, opts NetworkValid
205214
deletionSucceeded = true
206215
})
207216

208-
t.Run("ValidateVPCNotFound", func(t *testing.T) {
209-
vpc, err := v1.ValidateGetVPC(ctx, client, v1.GetVPCArgs{
210-
ID: vpcID,
217+
// Test #5: WaitForVPCToBeDeleted
218+
t.Run("WaitForVPCToBeDeleted", func(t *testing.T) {
219+
err := WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.VPC]{
220+
GetResource: func() (*v1.VPC, error) {
221+
return client.GetVPC(ctx, v1.GetVPCArgs{ID: vpcID})
222+
},
223+
Predicate: func(_ *v1.VPC) bool {
224+
return false // continue until failure
225+
},
226+
Timeout: 5 * time.Minute,
227+
Interval: 5 * time.Second,
211228
})
212-
require.Nil(t, vpc)
213229
require.ErrorIs(t, err, v1.ErrResourceNotFound)
230+
deletionSucceeded = true
214231
})
215232
}
216233

@@ -220,6 +237,8 @@ type KubernetesValidationOpts struct {
220237
KubernetesVersion string
221238
NodeGroupOpts *KubernetesValidationNodeGroupOpts
222239
NetworkOpts *KubernetesValidationNetworkOpts
240+
UserOpts *KubernetesValidationUserOpts
241+
Tags map[string]string
223242
}
224243

225244
type KubernetesValidationNodeGroupOpts struct {
@@ -240,12 +259,18 @@ type KubernetesValidationNetworkOpts struct {
240259
PrivateSubnetCidrBlock string
241260
}
242261

243-
func RunKubernetesValidation(t *testing.T, config ProviderConfig, opts KubernetesValidationOpts) {
262+
type KubernetesValidationUserOpts struct {
263+
Username string
264+
Role string
265+
RSAPEMBase64 string
266+
}
267+
268+
func RunKubernetesValidation(t *testing.T, config ProviderConfig, opts KubernetesValidationOpts) { //nolint:funlen // This function is long but it is a validation suite
244269
if testing.Short() {
245270
t.Skip("Skipping validation tests in short mode")
246271
}
247272

248-
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
273+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
249274
defer cancel()
250275

251276
client, err := config.Credential.MakeClient(ctx, config.Location)
@@ -267,33 +292,38 @@ func RunKubernetesValidation(t *testing.T, config ProviderConfig, opts Kubernete
267292
{CidrBlock: opts.NetworkOpts.PublicSubnetCidrBlock, Type: v1.SubnetTypePublic},
268293
{CidrBlock: opts.NetworkOpts.PrivateSubnetCidrBlock, Type: v1.SubnetTypePrivate},
269294
},
295+
Tags: opts.Tags,
270296
})
271297
require.NoError(t, err, "ValidateCreateVPC should pass")
272298

273299
// Wait for the VPC to be available
274-
err = v1.WaitForVPCPredicate(ctx, client, v1.GetVPCArgs{ID: vpc.ID},
275-
v1.WaitForVPCPredicateOpts{
276-
Predicate: func(vpc *v1.VPC) bool {
277-
return vpc.Status == v1.VPCStatusAvailable
278-
},
279-
Timeout: 5 * time.Minute,
280-
Interval: 5 * time.Second,
300+
err = WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.VPC]{
301+
GetResource: func() (*v1.VPC, error) {
302+
return client.GetVPC(ctx, v1.GetVPCArgs{ID: vpc.ID})
281303
},
282-
)
304+
Predicate: func(vpc *v1.VPC) bool {
305+
return vpc.Status == v1.VPCStatusAvailable
306+
},
307+
Timeout: 5 * time.Minute,
308+
Interval: 5 * time.Second,
309+
})
283310
require.NoError(t, err, "WaitForVPCToBeAvailable should pass")
311+
t.Logf("VPC created: %s", vpc.ID)
284312

313+
// The VPC was created successfully -- create a defer function to delete the VPC if the tests fail
285314
defer func() {
286-
// Clean up the VPC if it was created
287315
if vpc != nil {
288316
err = v1.ValidateDeleteVPC(ctx, client, v1.DeleteVPCArgs{
289317
ID: vpc.ID,
290318
})
291319
if err != nil {
292320
t.Fatalf("Failed to cleanup after validation of VPC: %v", err)
293321
}
322+
t.Logf("VPC deleted: %s", vpc.ID)
294323
}
295324
}()
296325

326+
// Test: Create Kubernetes Cluster
297327
var clusterID v1.CloudProviderResourceID
298328
t.Run("ValidateCreateKubernetesCluster", func(t *testing.T) {
299329
cluster, err := v1.ValidateCreateKubernetesCluster(ctx, client, v1.CreateClusterArgs{
@@ -303,12 +333,28 @@ func RunKubernetesValidation(t *testing.T, config ProviderConfig, opts Kubernete
303333
SubnetIDs: []v1.CloudProviderResourceID{vpc.Subnets[0].ID},
304334
KubernetesVersion: opts.KubernetesVersion,
305335
Location: opts.NetworkOpts.Location,
336+
Tags: opts.Tags,
306337
})
307338
require.NoError(t, err, "ValidateCreateKubernetesCluster should pass")
308339
require.NotNil(t, cluster)
309340
clusterID = cluster.ID
310341
})
311342

343+
// The Kubernetes cluster was created successfully -- create a defer function to delete the Kubernetes cluster if the tests fail
344+
clusterDeletionSucceeded := false
345+
defer func() {
346+
if !clusterDeletionSucceeded && clusterID != "" {
347+
t.Logf("Cleaning up Kubernetes cluster after failed tests: %s", clusterID)
348+
err = v1.ValidateDeleteKubernetesCluster(ctx, client, v1.DeleteClusterArgs{
349+
ID: clusterID,
350+
})
351+
if err != nil {
352+
t.Fatalf("Failed to cleanup after validation of Kubernetes cluster: %v", err)
353+
}
354+
}
355+
}()
356+
357+
// Test: Get Kubernetes Cluster
312358
t.Run("ValidateGetKubernetesCluster", func(t *testing.T) {
313359
cluster, err := v1.ValidateGetKubernetesCluster(ctx, client, v1.GetClusterArgs{
314360
ID: clusterID,
@@ -317,63 +363,115 @@ func RunKubernetesValidation(t *testing.T, config ProviderConfig, opts Kubernete
317363
require.NotNil(t, cluster)
318364
})
319365

366+
// Test: WaitFor Kubernetes Cluster to Be Available
320367
t.Run("WaitForKubernetesClusterToBeAvailable", func(t *testing.T) {
321-
err := v1.WaitForKubernetesClusterPredicate(ctx, client, v1.GetClusterArgs{ID: clusterID},
322-
v1.WaitForKubernetesClusterPredicateOpts{
323-
Predicate: func(cluster *v1.Cluster) bool {
324-
return cluster.Status == v1.ClusterStatusAvailable
325-
},
326-
Timeout: 20 * time.Minute,
327-
Interval: 15 * time.Second,
368+
err := WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.Cluster]{
369+
GetResource: func() (*v1.Cluster, error) {
370+
return client.GetCluster(ctx, v1.GetClusterArgs{ID: clusterID})
328371
},
329-
)
372+
Predicate: func(cluster *v1.Cluster) bool {
373+
return cluster.Status == v1.ClusterStatusAvailable
374+
},
375+
Timeout: 20 * time.Minute,
376+
Interval: 15 * time.Second,
377+
})
330378
require.NoError(t, err, "WaitForKubernetesClusterToBeAvailable should pass")
331379
})
332380

333-
// t.Run("ValidateGetKubernetesClusterCredentials", func(t *testing.T) {
334-
// err := v1.ValidateGetKubernetesClusterCredentials(ctx, client, v1.GetClusterArgs{
335-
// ID: v1.CloudProviderResourceID("test-cluster"),
336-
// })
337-
// require.NoError(t, err, "ValidateGetKubernetesClusterCredentials should pass")
338-
// })
339-
340-
// t.Run("ValidateCreateKubernetesNodeGroup", func(t *testing.T) {
341-
// err := v1.ValidateCreateKubernetesNodeGroup(ctx, client, v1.CreateNodeGroupArgs{
342-
// ClusterID: v1.CloudProviderResourceID("test-cluster"),
343-
// Name: "test-node-group",
344-
// RefID: "test-node-group",
345-
// MinNodeCount: 1,
346-
// MaxNodeCount: 1,
347-
// InstanceType: "test-instance-type",
348-
// DiskSizeGiB: 100,
349-
// })
350-
// require.NoError(t, err, "ValidateCreateKubernetesNodeGroup should pass")
351-
// })
352-
353-
// t.Run("ValidateDeleteKubernetesNodeGroup", func(t *testing.T) {
354-
// err := v1.ValidateDeleteKubernetesNodeGroup(ctx, client, v1.DeleteNodeGroupArgs{
355-
// ID: v1.CloudProviderResourceID("test-node-group"),
356-
// })
357-
// require.NoError(t, err, "ValidateDeleteKubernetesNodeGroup should pass")
358-
// })
381+
// Test: Get Kubernetes Cluster Credentials
382+
t.Run("ValidateGetKubernetesClusterCredentials", func(t *testing.T) {
383+
_, err := v1.ValidateGetKubernetesClusterCredentials(ctx, client, v1.PutUserArgs{
384+
ClusterID: clusterID,
385+
Username: opts.UserOpts.Username,
386+
Role: opts.UserOpts.Role,
387+
RSAPEMBase64: opts.UserOpts.RSAPEMBase64,
388+
})
389+
require.NoError(t, err, "ValidateGetKubernetesClusterCredentials should pass")
390+
})
391+
392+
// Test: Create Kubernetes Node Group
393+
var nodeGroup v1.NodeGroup
394+
t.Run("ValidateCreateKubernetesNodeGroup", func(t *testing.T) {
395+
ng, err := v1.ValidateCreateKubernetesNodeGroup(ctx, client, v1.CreateNodeGroupArgs{
396+
ClusterID: clusterID,
397+
Name: opts.NodeGroupOpts.Name,
398+
RefID: opts.NodeGroupOpts.RefID,
399+
MinNodeCount: opts.NodeGroupOpts.MinNodeCount,
400+
MaxNodeCount: opts.NodeGroupOpts.MaxNodeCount,
401+
InstanceType: opts.NodeGroupOpts.InstanceType,
402+
DiskSizeGiB: opts.NodeGroupOpts.DiskSizeGiB,
403+
Tags: opts.Tags,
404+
})
405+
require.NoError(t, err, "ValidateCreateKubernetesNodeGroup should pass")
406+
require.NotNil(t, ng)
407+
nodeGroup = *ng
408+
})
409+
410+
// The node group was created successfully -- create a defer function to delete the node group if the tests fail
411+
nodeGroupDeletionSucceeded := false
412+
defer func() {
413+
if !nodeGroupDeletionSucceeded && nodeGroup.ID != "" {
414+
t.Logf("Cleaning up Kubernetes node group after failed tests: %s", nodeGroup.ID)
415+
err = v1.ValidateDeleteKubernetesNodeGroup(ctx, client, v1.DeleteNodeGroupArgs{
416+
ID: nodeGroup.ID,
417+
})
418+
if err != nil {
419+
t.Fatalf("Failed to cleanup after validation of Kubernetes node group: %v", err)
420+
}
421+
}
422+
}()
423+
424+
// Test: WaitFor Kubernetes Node Group to Be Available
425+
t.Run("WaitForKubernetesNodeGroupToBeAvailable", func(t *testing.T) {
426+
err := WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.NodeGroup]{
427+
GetResource: func() (*v1.NodeGroup, error) {
428+
return client.GetNodeGroup(ctx, v1.GetNodeGroupArgs{ID: nodeGroup.ID})
429+
},
430+
Predicate: func(nodeGroup *v1.NodeGroup) bool {
431+
return nodeGroup.Status == v1.NodeGroupStatusAvailable
432+
},
433+
Timeout: 5 * time.Minute,
434+
Interval: 5 * time.Second,
435+
})
436+
require.NoError(t, err, "WaitForKubernetesNodeGroupToBeAvailable should pass")
437+
})
438+
439+
// Test: Validate Cluster Node Groups matches the created node group
440+
t.Run("ValidateClusterNodeGroups", func(t *testing.T) {
441+
err := v1.ValidateClusterNodeGroups(ctx, client, v1.GetClusterArgs{ID: clusterID}, nodeGroup)
442+
require.NoError(t, err, "ValidateClusterNodeGroups should pass")
443+
})
359444

445+
// Test: Delete Kubernetes Node Group
446+
t.Run("ValidateDeleteKubernetesNodeGroup", func(t *testing.T) {
447+
err := v1.ValidateDeleteKubernetesNodeGroup(ctx, client, v1.DeleteNodeGroupArgs{
448+
ID: nodeGroup.ID,
449+
})
450+
require.NoError(t, err, "ValidateDeleteKubernetesNodeGroup should pass")
451+
nodeGroupDeletionSucceeded = true
452+
})
453+
454+
// Test: Delete Kubernetes Cluster
360455
t.Run("ValidateDeleteKubernetesCluster", func(t *testing.T) {
361456
err := v1.ValidateDeleteKubernetesCluster(ctx, client, v1.DeleteClusterArgs{
362457
ID: clusterID,
363458
})
364459
require.NoError(t, err, "ValidateDeleteKubernetesCluster should pass")
365460
})
366461

462+
// Test: WaitFor Kubernetes Cluster to Be Deleted
367463
t.Run("WaitForKubernetesClusterToBeDeleted", func(t *testing.T) {
368-
err := v1.WaitForKubernetesClusterPredicate(ctx, client, v1.GetClusterArgs{ID: clusterID},
369-
v1.WaitForKubernetesClusterPredicateOpts{
370-
Predicate: func(_ *v1.Cluster) bool {
371-
return false // continue until failure
372-
},
373-
Timeout: 5 * time.Minute,
374-
Interval: 5 * time.Second,
464+
err := WaitForResourcePredicate(ctx, WaitForResourcePredicateOpts[*v1.Cluster]{
465+
GetResource: func() (*v1.Cluster, error) {
466+
return client.GetCluster(ctx, v1.GetClusterArgs{ID: clusterID})
467+
},
468+
Predicate: func(_ *v1.Cluster) bool {
469+
return false // continue until failure
375470
},
376-
)
471+
Timeout: 5 * time.Minute,
472+
Interval: 5 * time.Second,
473+
})
377474
require.ErrorIs(t, err, v1.ErrResourceNotFound)
475+
clusterDeletionSucceeded = true
378476
})
379477
}

0 commit comments

Comments
 (0)