Skip to content

Commit 2dfb81e

Browse files
a-r-ncopybara-github
authored andcommitted
networkutils: update RDMA network creation
- Only use a single MRDMA net/subnet for each MRDMA NIC. - Handle the unique naming scheme and stack requirements for `metal` RDMA networks. PiperOrigin-RevId: 951585553
1 parent 8cc1ddc commit 2dfb81e

2 files changed

Lines changed: 104 additions & 42 deletions

File tree

utils/networkutils/networkutils.go

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -244,20 +244,20 @@ func daisyNetworkForIRDMANIC(index int, project string, zone string, isMetal boo
244244
Network: compute.Network{
245245
Name: fmt.Sprintf("irdma-network-%d", index),
246246
Mtu: int64(imagetest.JumboFramesMTU),
247-
NetworkProfile: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networkProfiles/%s-vpc-falcon", project, zone),
247+
NetworkProfile: fmt.Sprintf("projects/%s/global/networkProfiles/%s-vpc-falcon", project, zone),
248248
},
249249
AutoCreateSubnetworks: new(bool),
250250
}
251251
}
252252

253253
func daisyNetworkForMRDMANIC(index int, project string, zone string, isMetal bool) *daisy.Network {
254-
networkProfile := fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networkProfiles/%s-vpc-roce", project, zone)
254+
networkProfile := fmt.Sprintf("projects/%s/global/networkProfiles/%s-vpc-roce", project, zone)
255255
if isMetal {
256256
networkProfile += "-metal"
257257
}
258258
return &daisy.Network{
259259
Network: compute.Network{
260-
Name: fmt.Sprintf("mrdma-network-%d", index),
260+
Name: "mrdma-network",
261261
Mtu: int64(imagetest.JumboFramesMTU),
262262
NetworkProfile: networkProfile,
263263
},
@@ -297,13 +297,27 @@ func regionFromZone(zone string) (string, error) {
297297
return strings.Join(parts[:2], "-"), nil
298298
}
299299

300-
func daisySubnet(index int, zone string) (*daisy.Subnetwork, error) {
301-
netPrefix, err := subnetPrefix(index)
300+
func daisySubnet(daisyNetwork *daisy.Network, index int, zone string) (*daisy.Subnetwork, error) {
301+
region, err := regionFromZone(zone)
302302
if err != nil {
303303
return nil, err
304304
}
305305

306-
region, err := regionFromZone(zone)
306+
if strings.HasSuffix(daisyNetwork.NetworkProfile, "-metal") {
307+
return &daisy.Subnetwork{
308+
Subnetwork: compute.Subnetwork{
309+
Name: fmt.Sprintf("default-subnet-1-%s", daisyNetwork.Name),
310+
StackType: "IPV6_ONLY",
311+
Ipv6AccessType: "INTERNAL",
312+
Region: region,
313+
},
314+
Resource: daisy.Resource{
315+
NoCleanup: true,
316+
},
317+
}, nil
318+
}
319+
320+
netPrefix, err := subnetPrefix(index)
307321
if err != nil {
308322
return nil, err
309323
}
@@ -341,32 +355,43 @@ type CreateMachineWithNetworksOptions struct {
341355
// It registers the networks and subnetwork creations with the test workflow.
342356
func CreateMachineWithNetworks(t *imagetest.TestWorkflow, o *CreateMachineWithNetworksOptions) (*daisy.Instance, error) {
343357
m := &daisy.Instance{}
358+
createdNetworks := make(map[string]*imagetest.Network)
359+
createdSubnetworks := make(map[string]bool)
344360

345361
for nicIndex, nicType := range o.NicTypes {
346362
daisyNetwork, err := daisyNetworkForNIC(nicType, nicIndex, o.Project, o.Zone, imagetest.IsMetal(o.MachineType))
347363
if err != nil {
348364
return nil, fmt.Errorf("building daisy network: %w", err)
349365
}
350366

351-
daisySubnet, err := daisySubnet(nicIndex, o.Zone)
352-
if err != nil {
353-
return nil, fmt.Errorf("building daisy subnetwork: %w", err)
367+
citNetwork, ok := createdNetworks[daisyNetwork.Name]
368+
if !ok {
369+
citNetwork, err = t.CreateNetworkFromDaisyNetwork(daisyNetwork)
370+
if err != nil {
371+
return nil, fmt.Errorf("creating network: %w", err)
372+
}
373+
createdNetworks[daisyNetwork.Name] = citNetwork
354374
}
355375

356-
citNetwork, err := t.CreateNetworkFromDaisyNetwork(daisyNetwork)
376+
daisySub, err := daisySubnet(daisyNetwork, nicIndex, o.Zone)
357377
if err != nil {
358-
return nil, fmt.Errorf("creating network: %w", err)
378+
return nil, fmt.Errorf("building daisy subnetwork: %w", err)
359379
}
360380

361-
if _, err = citNetwork.CreateSubnetworkFromDaisySubnetwork(daisySubnet); err != nil {
362-
return nil, fmt.Errorf("creating subnetwork: %w", err)
381+
if !createdSubnetworks[daisySub.Name] {
382+
if _, err = citNetwork.CreateSubnetworkFromDaisySubnetwork(daisySub); err != nil {
383+
return nil, fmt.Errorf("creating subnetwork: %w", err)
384+
}
385+
createdSubnetworks[daisySub.Name] = true
363386
}
364387

365388
m.NetworkInterfaces = append(m.NetworkInterfaces, &compute.NetworkInterface{
366-
NicType: nicType,
367-
Network: daisyNetwork.Name,
368-
Subnetwork: daisySubnet.Name,
369-
AccessConfigs: accessConfigsForNIC(nicType),
389+
NicType: nicType,
390+
Network: daisyNetwork.Name,
391+
Subnetwork: daisySub.Name,
392+
StackType: daisySub.StackType,
393+
Ipv6AccessType: daisySub.Ipv6AccessType,
394+
AccessConfigs: accessConfigsForNIC(nicType),
370395
})
371396
}
372397

utils/networkutils/networkutils_test.go

Lines changed: 62 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ func TestDaisyNetworkForNIC(t *testing.T) {
481481
Network: compute.Network{
482482
Name: "irdma-network-0",
483483
Mtu: 8896,
484-
NetworkProfile: "https://www.googleapis.com/compute/v1/projects/test-project/global/networkProfiles/us-central1-a-vpc-falcon",
484+
NetworkProfile: "projects/test-project/global/networkProfiles/us-central1-a-vpc-falcon",
485485
},
486486
AutoCreateSubnetworks: new(bool),
487487
},
@@ -494,9 +494,9 @@ func TestDaisyNetworkForNIC(t *testing.T) {
494494
zone: "europe-central2-a",
495495
want: &daisy.Network{
496496
Network: compute.Network{
497-
Name: "mrdma-network-0",
497+
Name: "mrdma-network",
498498
Mtu: 8896,
499-
NetworkProfile: "https://www.googleapis.com/compute/v1/projects/test-project/global/networkProfiles/europe-central2-a-vpc-roce",
499+
NetworkProfile: "projects/test-project/global/networkProfiles/europe-central2-a-vpc-roce",
500500
},
501501
AutoCreateSubnetworks: new(bool),
502502
},
@@ -510,9 +510,9 @@ func TestDaisyNetworkForNIC(t *testing.T) {
510510
isMetal: true,
511511
want: &daisy.Network{
512512
Network: compute.Network{
513-
Name: "mrdma-network-0",
513+
Name: "mrdma-network",
514514
Mtu: 8896,
515-
NetworkProfile: "https://www.googleapis.com/compute/v1/projects/test-project/global/networkProfiles/asia-southeast2-b-vpc-roce-metal",
515+
NetworkProfile: "projects/test-project/global/networkProfiles/asia-southeast2-b-vpc-roce-metal",
516516
},
517517
AutoCreateSubnetworks: new(bool),
518518
},
@@ -534,16 +534,18 @@ func TestDaisyNetworkForNIC(t *testing.T) {
534534

535535
func TestDaisySubnet(t *testing.T) {
536536
cases := []struct {
537-
name string
538-
index int
539-
zone string
540-
want *daisy.Subnetwork
541-
wantErr bool
537+
name string
538+
daisyNetwork *daisy.Network
539+
index int
540+
zone string
541+
want *daisy.Subnetwork
542+
wantErr bool
542543
}{
543544
{
544-
name: "index 0",
545-
index: 0,
546-
zone: "us-central1-a",
545+
name: "general purpose subnet",
546+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "net-0"}},
547+
index: 0,
548+
zone: "us-central1-a",
547549
want: &daisy.Subnetwork{
548550
Subnetwork: compute.Subnetwork{
549551
Name: "subnet-0",
@@ -553,9 +555,10 @@ func TestDaisySubnet(t *testing.T) {
553555
},
554556
},
555557
{
556-
name: "index 1",
557-
index: 1,
558-
zone: "europe-west1-b",
558+
name: "index 1",
559+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "net-1"}},
560+
index: 1,
561+
zone: "europe-west1-b",
559562
want: &daisy.Subnetwork{
560563
Subnetwork: compute.Subnetwork{
561564
Name: "subnet-1",
@@ -565,26 +568,60 @@ func TestDaisySubnet(t *testing.T) {
565568
},
566569
},
567570
{
568-
name: "index too low",
569-
index: -1,
570-
wantErr: true,
571+
name: "MRDMA metal NetworkProfile subnet",
572+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "mrdma-network", NetworkProfile: "projects/p/global/networkProfiles/zone-vpc-roce-metal"}},
573+
index: 0,
574+
zone: "us-central1-a",
575+
want: &daisy.Subnetwork{
576+
Subnetwork: compute.Subnetwork{
577+
Name: "default-subnet-1-mrdma-network",
578+
StackType: "IPV6_ONLY",
579+
Ipv6AccessType: "INTERNAL",
580+
Region: "us-central1",
581+
},
582+
Resource: daisy.Resource{
583+
NoCleanup: true,
584+
},
585+
},
571586
},
572587
{
573-
name: "index too high",
574-
index: 256,
575-
wantErr: true,
588+
name: "MRDMA virtual NetworkProfile subnet",
589+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "mrdma-network", NetworkProfile: "projects/p/global/networkProfiles/zone-vpc-roce"}},
590+
index: 0,
591+
zone: "us-central1-a",
592+
want: &daisy.Subnetwork{
593+
Subnetwork: compute.Subnetwork{
594+
Name: "subnet-0",
595+
IpCidrRange: "10.0.0.0/24",
596+
Region: "us-central1",
597+
},
598+
},
599+
},
600+
{
601+
name: "index too low",
602+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "net-0"}},
603+
index: -1,
604+
zone: "us-central1-a",
605+
wantErr: true,
606+
},
607+
{
608+
name: "index too high",
609+
daisyNetwork: &daisy.Network{Network: compute.Network{Name: "net-0"}},
610+
index: 256,
611+
zone: "us-central1-a",
612+
wantErr: true,
576613
},
577614
}
578615

579616
for _, tc := range cases {
580617
t.Run(tc.name, func(t *testing.T) {
581-
got, err := daisySubnet(tc.index, tc.zone)
618+
got, err := daisySubnet(tc.daisyNetwork, tc.index, tc.zone)
582619
if (err != nil) != tc.wantErr {
583-
t.Errorf("daisySubnet(%d, %q) returned error %v, wantErr %t", tc.index, tc.zone, err, tc.wantErr)
620+
t.Errorf("daisySubnet(%v, %d, %q) returned error %v, wantErr %t", tc.daisyNetwork, tc.index, tc.zone, err, tc.wantErr)
584621
return
585622
}
586623
if diff := diffDaisy(got, tc.want); diff != "" {
587-
t.Errorf("daisySubnet(%d, %q) = doesn't match expectations: diff (-got +want):\n%s", tc.index, tc.zone, diff)
624+
t.Errorf("daisySubnet(%v, %d, %q) = doesn't match expectations: diff (-got +want):\n%s", tc.daisyNetwork, tc.index, tc.zone, diff)
588625
}
589626
})
590627
}

0 commit comments

Comments
 (0)