do not leak api-server cluster ip from seed to shoot (gardener#10949)

* fix: do not leak api-server cluster ip from seed to shoot

* Apply suggestions from code review

new lines etc.

Co-authored-by: Johannes Scheerer <johannes.scheerer@sap.com>
Co-authored-by: Rafael Franzke <rafael.franzke@sap.com>

* refactor: remove unused func

* refactor: remove unnecessary const

* refactor: move seed range check from shoot to cidr

* docs: add reserved ranges to shoot networking guide

* refactor: rename ReservedSeedServiceRange => ReservedKubeApiServerMappingRange

* feat: add IsIPv4, IsIPv6 helper functions to cidr

---------

Co-authored-by: Johannes Scheerer <johannes.scheerer@sap.com>
Co-authored-by: Rafael Franzke <rafael.franzke@sap.com>

Author: 3 people

docs/usage/networking/shoot_networking.md

@@ -85,3 +85,17 @@ Number of IPs per podCIDRs: 128
 ```
 
 With the configuration above, a Shoot cluster can at most have **32 nodes** which are ready to run workload in the Pod network.
+
+## Reserved Networks
+
+Some network ranges are reserved for specific use-cases in the communication between seeds and shoots.
+
+| IPv  | CIDR                  | Name                         | Purpose                                                                                                                              |
+|------|-----------------------|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| IPv4 | 192.168.123.0/24      | Default VPN Range            | Used for communication between seed API server and shoot resources via VPN. Will be removed once feature gate `NewVPN` is graduated. |
+| IPv6 | fd8f:6d53:b97a:1::/96 | Default VPN Range            |                                                                                                                                      |
+| IPv4 | 240.0.0.0/8           | Kube-ApiServer Mapping Range | Used for the `kubernetes.default.svc.cluster.local` service in a shoot                                                               |
+
+> :warning: Do not use any of the CIDR ranges mentioned above for any of the node, pod or service networks.
+> Gardener will prevent their creation. Pre-existing shoots using reserved ranges will still work, though it is recommended
+> to recreate them with compatible network ranges.

pkg/apis/core/v1beta1/constants/types_constants.go

@@ -781,6 +781,8 @@ const (
 	DefaultVPNRange = "192.168.123.0/24"
 	// DefaultVPNRangeV6 is the default IPv6 network range for the VPN between seed and shoot cluster.
 	DefaultVPNRangeV6 = "fd8f:6d53:b97a:1::/96"
+	// ReservedKubeApiServerMappingRange is the IPv4 network range for the "kubernetes" service used by apiserver-proxy
+	ReservedKubeApiServerMappingRange = "240.0.0.0/8"
 
 	// BackupSecretName is the name of secret having credentials for etcd backups.
 	BackupSecretName string = "etcd-backup"

pkg/apis/core/validation/seed.go

@@ -248,15 +248,22 @@ func validateSeedNetworks(seedNetworks core.SeedNetworks, fldPath *field.Path, i
 	}
 
 	var (
-		primaryIPFamily = helper.DeterminePrimaryIPFamily(seedNetworks.IPFamilies)
-		networks        []cidrvalidation.CIDR
+		primaryIPFamily          = helper.DeterminePrimaryIPFamily(seedNetworks.IPFamilies)
+		networks                 []cidrvalidation.CIDR
+		reservedSeedServiceRange = cidrvalidation.NewCIDR(v1beta1constants.ReservedKubeApiServerMappingRange, field.NewPath(""))
 	)
 
 	if !inTemplate || len(seedNetworks.Pods) > 0 {
 		networks = append(networks, cidrvalidation.NewCIDR(seedNetworks.Pods, fldPath.Child("pods")))
 	}
 	if !inTemplate || len(seedNetworks.Services) > 0 {
-		networks = append(networks, cidrvalidation.NewCIDR(seedNetworks.Services, fldPath.Child("services")))
+		services := cidrvalidation.NewCIDR(seedNetworks.Services, fldPath.Child("services"))
+		networks = append(networks, services)
+		// Service range must not be larger than /8 for ipv4
+		if services.IsIPv4() {
+			maxSize, _ := reservedSeedServiceRange.GetIPNet().Mask.Size()
+			allErrs = append(allErrs, services.ValidateMaxSize(maxSize)...)
+		}
 	}
 	if seedNetworks.Nodes != nil {
 		networks = append(networks, cidrvalidation.NewCIDR(*seedNetworks.Nodes, fldPath.Child("nodes")))
@@ -277,6 +284,7 @@ func validateSeedNetworks(seedNetworks core.SeedNetworks, fldPath *field.Path, i
 	}
 	allErrs = append(allErrs, cidrvalidation.ValidateCIDROverlap(networks, false)...)
 
+	allErrs = append(allErrs, reservedSeedServiceRange.ValidateNotOverlap(networks...)...)
 	vpnRange := cidrvalidation.NewCIDR(v1beta1constants.DefaultVPNRange, field.NewPath(""))
 	allErrs = append(allErrs, vpnRange.ValidateNotOverlap(networks...)...)
 	vpnRangeV6 := cidrvalidation.NewCIDR(v1beta1constants.DefaultVPNRangeV6, field.NewPath(""))

pkg/apis/core/validation/seed_test.go

@@ -535,6 +535,32 @@ var _ = Describe("Seed Validation Tests", func() {
 						"Detail": Equal(`must not overlap with "[]" ("192.168.123.0/24")`),
 					}))
 				})
+
+				It("should forbid Seed with overlap to reserved range", func() {
+					// Service CIDR overlaps with reserved range
+					seed.Spec.Networks.Pods = "240.0.0.0/16" // 240.0.0.0 -> 240.0.255.255
+
+					errorList := ValidateSeed(seed)
+
+					Expect(errorList).To(ConsistOfFields(Fields{
+						"Type":   Equal(field.ErrorTypeInvalid),
+						"Field":  Equal("spec.networks.pods"),
+						"Detail": Equal(`must not overlap with "[]" ("240.0.0.0/8")`),
+					}))
+				})
+
+				It("should forbid Seed with too large service range", func() {
+					// Service CIDR too large
+					seed.Spec.Networks.Services = "90.0.0.0/7" // 90.0.0.0 -> 91.255.255.255
+
+					errorList := ValidateSeed(seed)
+
+					Expect(errorList).To(ConsistOfFields(Fields{
+						"Type":   Equal(field.ErrorTypeInvalid),
+						"Field":  Equal("spec.networks.services"),
+						"Detail": Equal(`cannot be larger than /8`),
+					}))
+				})
 			})
 
 			Context("IPv6", func() {
@@ -616,6 +642,18 @@ var _ = Describe("Seed Validation Tests", func() {
 						"Detail": ContainSubstring("must be a valid IPv6 address"),
 					}))
 				})
+
+				It("should allow Seed with very large service range because the range limit only applies to ipv4", func() {
+					seed.Spec.Networks.Nodes = ptr.To("2001:db8:11::/48")
+					seed.Spec.Networks.Pods = "2001:db8:12::/48"
+					seed.Spec.Networks.Services = "3001::/7" // larger than /8 ipv4 limit
+					seed.Spec.Networks.ShootDefaults.Pods = ptr.To("2001:db8:1::/48")
+					seed.Spec.Networks.ShootDefaults.Services = ptr.To("2001:db8:3::/48")
+
+					errorList := ValidateSeed(seed)
+
+					Expect(errorList).To(BeEmpty())
+				})
 			})
 		})
 

pkg/gardenlet/operation/botanist/kubeapiserverexposure.go

@@ -68,11 +68,22 @@ func (b *Botanist) DeployKubeAPIServerSNI(ctx context.Context) error {
 }
 
 func (b *Botanist) setAPIServerServiceClusterIP(clusterIP string) {
-	if b.Shoot.GetInfo().Spec.Networking.IPFamilies[0] == v1beta1.IPFamilyIPv6 && net.ParseIP(clusterIP).To4() != nil {
-		// "64:ff9b:1::" is a well known prefix for address translation for use
-		// in local networks.
-		b.APIServerClusterIP = "64:ff9b:1::" + clusterIP
+	clusterIPv4 := net.ParseIP(clusterIP).To4()
+
+	if clusterIPv4 != nil {
+		if b.Shoot.GetInfo().Spec.Networking.IPFamilies[0] == v1beta1.IPFamilyIPv6 {
+			// "64:ff9b:1::" is a well known prefix for address translation for use
+			// in local networks.
+			b.APIServerClusterIP = "64:ff9b:1::" + clusterIP
+		} else {
+			// prevent leakage of real cluster ip to shoot. we use the reserved range 240.0.0.0/8 as prefix instead.
+			// e.g. cluster ip in seed:  192.168.102.23 => ip in shoot:  240.168.102.23
+			prefixIp, _, _ := net.ParseCIDR(v1beta1constants.ReservedKubeApiServerMappingRange)
+			prefix := prefixIp.To4()
+			b.APIServerClusterIP = net.IPv4(prefix[0], clusterIPv4[1], clusterIPv4[2], clusterIPv4[3]).String()
+		}
 	} else {
+		// regular ipv6 cluster ip
 		b.APIServerClusterIP = clusterIP
 	}
 	b.Shoot.Components.ControlPlane.KubeAPIServerSNI = kubeapiserverexposure.NewSNI(

pkg/utils/validation/cidr/cidr.go

@@ -41,6 +41,12 @@ type CIDR interface {
 	LastIPInRange() net.IP
 	// ValidateOverlap returns errors if the subnets do not overlap with CIDR.
 	ValidateOverlap(subsets ...CIDR) field.ErrorList
+	// ValidateMaxSize returns errors if the subnet is larger than the given bits. e.g. /15 is larger than /16
+	ValidateMaxSize(bits int) field.ErrorList
+	// IsIPv4 returns true if the CIDR is a valid v4 CIDR, false otherwise.
+	IsIPv4() bool
+	// IsIPv6 returns true if the CIDR is a valid v6 CIDR, false otherwise.
+	IsIPv6() bool
 }
 
 type cidrPath struct {
@@ -147,6 +153,22 @@ func (c *cidrPath) ValidateIPFamily(ipFamily string) field.ErrorList {
 	return allErrs
 }
 
+// ValidateMaxSize returns an error if CIDR size is larger than given bits. e.g. /15 is larger than /16
+func (c *cidrPath) ValidateMaxSize(bits int) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if c.ParseError != nil {
+		return allErrs
+	}
+	cidrBits, _ := c.net.Mask.Size()
+
+	if cidrBits < bits {
+		allErrs = append(allErrs, field.Invalid(c.fieldPath, c.net.String(), fmt.Sprintf("cannot be larger than /%d", bits)))
+	}
+
+	return allErrs
+}
+
 func (c *cidrPath) Parse() (success bool) {
 	return c.ParseError == nil
 }
@@ -176,3 +198,17 @@ func (c *cidrPath) LastIPInRange() net.IP {
 
 	return res
 }
+
+func (c *cidrPath) IsIPv4() bool {
+	if c.ParseError == nil && len(c.ValidateIPFamily(IPFamilyIPv4)) == 0 {
+		return true
+	}
+	return false
+}
+
+func (c *cidrPath) IsIPv6() bool {
+	if c.ParseError == nil && len(c.ValidateIPFamily(IPFamilyIPv6)) == 0 {
+		return true
+	}
+	return false
+}

pkg/utils/validation/cidr/cidr_test.go

@@ -274,6 +274,69 @@ var _ = Describe("cidr", func() {
 				Expect(other.ValidateOverlap(cdr)).To(BeEmpty())
 			})
 		})
+
+		Describe("ValidateMaxSize", func() {
+			It("should return no errors if cidr is same size than limit", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "11.12.0.0/16"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.ValidateMaxSize(16)).To(BeEmpty())
+			})
+
+			It("should return no errors if cidr is smaller than limit", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "11.12.0.0/17"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.ValidateMaxSize(16)).To(BeEmpty())
+			})
+
+			It("should return an error if cidr is larger than limit", func() {
+				badPath := field.NewPath("bad")
+				badCIDR := "11.12.0.0/15"
+				bad := NewCIDR(badCIDR, badPath)
+
+				Expect(bad.ValidateMaxSize(16)).To(ConsistOfFields(Fields{
+					"Type":     Equal(field.ErrorTypeInvalid),
+					"Field":    Equal(badPath.String()),
+					"BadValue": Equal(badCIDR),
+					"Detail":   Equal(`cannot be larger than /16`),
+				}))
+			})
+		})
+
+		Describe("IsIPv4", func() {
+			It("should return true on a valid IPv4 cidr", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "10.100.0.0/16"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.Parse()).To(BeTrue())
+				Expect(good.IsIPv4()).To(BeTrue())
+				Expect(good.IsIPv6()).To(BeFalse())
+			})
+
+			It("should return false on a malformed IPv4 cidr", func() {
+				badPath := field.NewPath("bad")
+				badCIDR := "10.100.0.0/123"
+				bad := NewCIDR(badCIDR, badPath)
+
+				Expect(bad.Parse()).To(BeFalse())
+				Expect(bad.IsIPv4()).To(BeFalse())
+				Expect(bad.IsIPv6()).To(BeFalse())
+			})
+
+			It("should return false on a valid IPv6 cidr", func() {
+				wrongPath := field.NewPath("wrong")
+				wrongCIDR := "2001:0db8::/16"
+				wrong := NewCIDR(wrongCIDR, wrongPath)
+
+				Expect(wrong.Parse()).To(BeTrue())
+				Expect(wrong.IsIPv4()).To(BeFalse())
+				Expect(wrong.IsIPv6()).To(BeTrue())
+			})
+		})
 	})
 
 	Context("IPv6", func() {
@@ -520,5 +583,68 @@ var _ = Describe("cidr", func() {
 				Expect(cdr.ValidateOverlap(other)).To(BeEmpty())
 			})
 		})
+
+		Describe("ValidateMaxSize", func() {
+			It("should return no errors if cidr is same size as limit", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "2001:1db8::/32"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.ValidateMaxSize(32)).To(BeEmpty())
+			})
+
+			It("should return no errors if cidr is smaller than limit", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "2001:1db8::/33"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.ValidateMaxSize(32)).To(BeEmpty())
+			})
+
+			It("should return an error if cidr is larger than limit", func() {
+				badPath := field.NewPath("bad")
+				badCIDR := "2001:1db8::/31"
+				bad := NewCIDR(badCIDR, badPath)
+
+				Expect(bad.ValidateMaxSize(32)).To(ConsistOfFields(Fields{
+					"Type":     Equal(field.ErrorTypeInvalid),
+					"Field":    Equal(badPath.String()),
+					"BadValue": Equal(badCIDR),
+					"Detail":   Equal(`cannot be larger than /32`),
+				}))
+			})
+		})
+
+		Describe("IsIPv6", func() {
+			It("should return true on a valid IPv6 cidr", func() {
+				goodPath := field.NewPath("good")
+				goodCIDR := "2001:0db8::/16"
+				good := NewCIDR(goodCIDR, goodPath)
+
+				Expect(good.Parse()).To(BeTrue())
+				Expect(good.IsIPv4()).To(BeFalse())
+				Expect(good.IsIPv6()).To(BeTrue())
+			})
+
+			It("should return false on a malformed IPv6 cidr", func() {
+				badPath := field.NewPath("bad")
+				badCIDR := "2001:0db8:xxyy:/64"
+				bad := NewCIDR(badCIDR, badPath)
+
+				Expect(bad.Parse()).To(BeFalse())
+				Expect(bad.IsIPv4()).To(BeFalse())
+				Expect(bad.IsIPv6()).To(BeFalse())
+			})
+
+			It("should return false on a valid IPv4 cidr", func() {
+				wrongPath := field.NewPath("wrong")
+				wrongCIDR := "10.100.0.0/16"
+				wrong := NewCIDR(wrongCIDR, wrongPath)
+
+				Expect(wrong.Parse()).To(BeTrue())
+				Expect(wrong.IsIPv4()).To(BeTrue())
+				Expect(wrong.IsIPv6()).To(BeFalse())
+			})
+		})
 	})
 })

pkg/utils/validation/cidr/disjoint.go

@@ -65,6 +65,10 @@ func validateOverlapWithSeed(fldPath *field.Path, shootNetwork []string, network
 		if NetworksIntersect(v1beta1constants.DefaultVPNRangeV6, network) {
 			allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with default vpn network (%s)", networkType, v1beta1constants.DefaultVPNRangeV6)))
 		}
+
+		if NetworksIntersect(v1beta1constants.ReservedKubeApiServerMappingRange, network) {
+			allErrs = append(allErrs, field.Invalid(fldPath, network, fmt.Sprintf("shoot %s network intersects with reserved kube-apiserver mapping range (%s)", networkType, v1beta1constants.ReservedKubeApiServerMappingRange)))
+		}
 	}
 	if len(shootNetwork) == 0 && networkRequired {
 		allErrs = append(allErrs, field.Required(fldPath, networkType+"s is required"))