diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 3cd81f004b55a..70df3133327b6 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -954,6 +954,40 @@ spec: description: ConfigOverride is the complete containerd config file provided by the user. type: string + gvisor: + description: GVisor configures the gVisor (runsc) sandboxed runtime. + properties: + enabled: + description: Enabled determines if kOps will install the gVisor + runtime. + type: boolean + packages: + description: Packages overrides the URL and hash for the gVisor + packages. + properties: + hashAmd64: + description: HashAmd64 overrides the hash for the AMD64 + package. + type: string + hashArm64: + description: HashArm64 overrides the hash for the ARM64 + package. + type: string + urlAmd64: + description: UrlAmd64 overrides the URL for the AMD64 + package. + type: string + urlArm64: + description: UrlArm64 overrides the URL for the ARM64 + package. + type: string + type: object + platform: + description: |- + Platform is the gVisor execution platform: "systrap" (default, works + everywhere including VMs) or "kvm" (bare-metal with KVM support). + type: string + type: object installCriCtl: description: InstallCriCtl installs crictl (default "false"). type: boolean diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 62b7f091416f0..1ac8f369eb0fc 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -136,6 +136,40 @@ spec: description: ConfigOverride is the complete containerd config file provided by the user. type: string + gvisor: + description: GVisor configures the gVisor (runsc) sandboxed runtime. + properties: + enabled: + description: Enabled determines if kOps will install the gVisor + runtime. + type: boolean + packages: + description: Packages overrides the URL and hash for the gVisor + packages. + properties: + hashAmd64: + description: HashAmd64 overrides the hash for the AMD64 + package. + type: string + hashArm64: + description: HashArm64 overrides the hash for the ARM64 + package. + type: string + urlAmd64: + description: UrlAmd64 overrides the URL for the AMD64 + package. + type: string + urlArm64: + description: UrlArm64 overrides the URL for the ARM64 + package. + type: string + type: object + platform: + description: |- + Platform is the gVisor execution platform: "systrap" (default, works + everywhere including VMs) or "kvm" (bare-metal with KVM support). + type: string + type: object installCriCtl: description: InstallCriCtl installs crictl (default "false"). type: boolean diff --git a/nodeup/pkg/model/containerd.go b/nodeup/pkg/model/containerd.go index 92636976d5918..1d6042b5988d6 100644 --- a/nodeup/pkg/model/containerd.go +++ b/nodeup/pkg/model/containerd.go @@ -93,6 +93,11 @@ func (b *ContainerdBuilder) Build(c *fi.NodeupModelBuilderContext) error { return err } + // If gVisor is enabled, emit the runsc shim config file + if b.InstallGVisorRuntime() { + b.buildGVisorShimConfig(c) + } + if installContainerd { if err := b.installContainerd(c); err != nil { return err @@ -564,6 +569,12 @@ func (b *ContainerdBuilder) buildContainerdConfigV2() (string, error) { } } + if b.InstallGVisorRuntime() { + if err := appendGVisorRuntimeConfig(config, []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}); err != nil { + return "", fmt.Errorf("appending gvisor runtime to v2 containerd config: %w", err) + } + } + if err := applyConfigAdditions(config, containerd.ConfigAdditions); err != nil { return "", fmt.Errorf("applying ConfigAdditions to v2 containerd config: %w", err) } @@ -617,6 +628,12 @@ func (b *ContainerdBuilder) buildContainerdConfigV3() (string, error) { } } + if b.InstallGVisorRuntime() { + if err := appendGVisorRuntimeConfig(config, []string{"plugins", "io.containerd.cri.v1.runtime", "containerd", "runtimes"}); err != nil { + return "", fmt.Errorf("appending gvisor runtime to v3 containerd config: %w", err) + } + } + if err := applyConfigAdditions(config, containerd.ConfigAdditions); err != nil { return "", fmt.Errorf("applying ConfigAdditions to v3 containerd config: %w", err) } @@ -691,6 +708,45 @@ func appendNvidiaGPURuntimeConfig(config *toml.Tree, runtimesPath []string) erro return nil } +// appendGVisorRuntimeConfig adds the "runsc" runtime entry under runtimesPath. +// runtimesPath is schema-specific so the same helper can serve both v2 and v3 builders. +func appendGVisorRuntimeConfig(config *toml.Tree, runtimesPath []string) error { + gvisorConfig, err := toml.TreeFromMap( + map[string]interface{}{ + "runtime_type": "io.containerd.runsc.v1", + }, + ) + if err != nil { + return err + } + + path := make([]string, len(runtimesPath)+1) + copy(path, runtimesPath) + path[len(runtimesPath)] = "runsc" + config.SetPath(path, gvisorConfig) + + return nil +} + +// buildGVisorShimConfig emits /etc/containerd/runsc.toml, the shim-level +// configuration consumed by containerd-shim-runsc-v1 at container creation. +// See https://gvisor.dev/docs/user_guide/containerd/configuration/ +func (b *ContainerdBuilder) buildGVisorShimConfig(c *fi.NodeupModelBuilderContext) { + platform := b.NodeupConfig.GVisor.Platform + if platform == "" { + platform = "systrap" + } + + shimConfig, _ := toml.Load("") + shimConfig.SetPath([]string{"runsc_config", "platform"}, platform) + + c.AddTask(&nodetasks.File{ + Path: "/etc/containerd/runsc.toml", + Contents: fi.NewStringResource(shimConfig.String()), + Type: nodetasks.FileType_File, + }) +} + // buildRegistryHosts emits one hosts.toml per RegistryMirrors entry under containerdRegistryDirPath. // The directory is referenced by registry.config_path in the main containerd config; the // emit-files-iff-mirrors-non-empty condition here must stay in sync with the registry.config_path diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index cd495f7df6fa6..a27bffb4958cb 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -563,6 +563,14 @@ func (c *NodeupModelContext) InstallNvidiaRuntime() bool { c.GPUVendor == architectures.GPUVendorNvidia } +// InstallGVisorRuntime returns true if the gVisor (runsc) runtime should be installed. +// gVisor is only supported on Debian-family distributions (Debian, Ubuntu). +func (c *NodeupModelContext) InstallGVisorRuntime() bool { + return c.NodeupConfig.GVisor != nil && + fi.ValueOf(c.NodeupConfig.GVisor.Enabled) && + c.Distribution.IsDebianFamily() +} + // CloudProvider returns the cloud provider we are running on func (c *NodeupModelContext) CloudProvider() kops.CloudProviderID { return c.BootConfig.CloudProvider diff --git a/nodeup/pkg/model/gvisor.go b/nodeup/pkg/model/gvisor.go new file mode 100644 index 0000000000000..e13ea9d1411e5 --- /dev/null +++ b/nodeup/pkg/model/gvisor.go @@ -0,0 +1,49 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// GVisorBuilder installs the gVisor (runsc) sandboxed runtime. +// Only supported on Debian-family distributions. +type GVisorBuilder struct { + *NodeupModelContext +} + +var _ fi.NodeupModelBuilder = &GVisorBuilder{} + +// Build installs gVisor packages via the upstream apt repository. +func (b *GVisorBuilder) Build(c *fi.NodeupModelBuilderContext) error { + if !b.InstallGVisorRuntime() { + return nil + } + + c.AddTask(&nodetasks.AptSource{ + Name: "gvisor", + Keyring: "https://gvisor.dev/archive.key", + Sources: []string{ + "deb [arch=$(dpkg --print-architecture)] https://storage.googleapis.com/gvisor/releases release main", + }, + }) + // The runsc package bundles both runsc and containerd-shim-runsc-v1. + c.AddTask(&nodetasks.Package{Name: "runsc"}) + + return nil +} diff --git a/pkg/apis/kops/containerdconfig.go b/pkg/apis/kops/containerdconfig.go index 6f34be07be60c..b6d87c5ef8158 100644 --- a/pkg/apis/kops/containerdconfig.go +++ b/pkg/apis/kops/containerdconfig.go @@ -26,6 +26,9 @@ const ( NvidiaDefaultDriverPackage = "nvidia-driver-535-server" // NvidiaDevicePluginImage is the Nvidia K8s device plugin container image NvidiaDevicePluginImage = "nvcr.io/nvidia/k8s-device-plugin:v0.17.3" + // GVisorDefaultPlatform is the default gVisor execution platform. + // systrap uses SECCOMP_RET_TRAP/SIGSYS and works in all environments including VMs. + GVisorDefaultPlatform = "systrap" ) // ContainerdConfig is the configuration for containerd @@ -55,6 +58,8 @@ type ContainerdConfig struct { Version *string `json:"version,omitempty"` // NvidiaGPU configures the Nvidia GPU runtime. NvidiaGPU *NvidiaGPUConfig `json:"nvidiaGPU,omitempty"` + // GVisor configures the gVisor (runsc) sandboxed runtime. + GVisor *GVisorConfig `json:"gvisor,omitempty"` // Runc configures the runc runtime. Runc *Runc `json:"runc,omitempty"` // SelinuxEnabled enables SELinux support @@ -106,3 +111,17 @@ type Runc struct { // Packages overrides the URL and hash for the packages. Packages *PackagesConfig `json:"packages,omitempty"` } + +// GVisorConfig configures the gVisor sandboxed container runtime. +// When enabled, kOps installs runsc and containerd-shim-runsc-v1, +// registers the "runsc" runtime handler in containerd, and deploys +// a Kubernetes RuntimeClass named "gvisor". +type GVisorConfig struct { + // Enabled determines if kOps will install the gVisor runtime. + Enabled *bool `json:"enabled,omitempty"` + // Platform is the gVisor execution platform: "systrap" (default, works + // everywhere including VMs) or "kvm" (bare-metal with KVM support). + Platform string `json:"platform,omitempty"` + // Packages overrides the URL and hash for the gVisor packages. + Packages *PackagesConfig `json:"packages,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/containerdconfig.go b/pkg/apis/kops/v1alpha2/containerdconfig.go index fd90d8a53a048..bf73b9b7e6862 100644 --- a/pkg/apis/kops/v1alpha2/containerdconfig.go +++ b/pkg/apis/kops/v1alpha2/containerdconfig.go @@ -48,6 +48,8 @@ type ContainerdConfig struct { Version *string `json:"version,omitempty"` // NvidiaGPU configures the Nvidia GPU runtime. NvidiaGPU *NvidiaGPUConfig `json:"nvidiaGPU,omitempty"` + // GVisor configures the gVisor (runsc) sandboxed runtime. + GVisor *GVisorConfig `json:"gvisor,omitempty"` // Runc configures the runc runtime. Runc *Runc `json:"runc,omitempty"` // SelinuxEnabled enables SELinux support @@ -99,3 +101,17 @@ type Runc struct { // Packages overrides the URL and hash for the packages. Packages *PackagesConfig `json:"packages,omitempty"` } + +// GVisorConfig configures the gVisor sandboxed container runtime. +// When enabled, kOps installs runsc and containerd-shim-runsc-v1, +// registers the "runsc" runtime handler in containerd, and deploys +// a Kubernetes RuntimeClass named "gvisor". +type GVisorConfig struct { + // Enabled determines if kOps will install the gVisor runtime. + Enabled *bool `json:"enabled,omitempty"` + // Platform is the gVisor execution platform: "systrap" (default, works + // everywhere including VMs) or "kvm" (bare-metal with KVM support). + Platform string `json:"platform,omitempty"` + // Packages overrides the URL and hash for the gVisor packages. + Packages *PackagesConfig `json:"packages,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 0c1f2862a2871..be90c4387d944 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -494,6 +494,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*GVisorConfig)(nil), (*kops.GVisorConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_GVisorConfig_To_kops_GVisorConfig(a.(*GVisorConfig), b.(*kops.GVisorConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.GVisorConfig)(nil), (*GVisorConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_GVisorConfig_To_v1alpha2_GVisorConfig(a.(*kops.GVisorConfig), b.(*GVisorConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*GossipConfig)(nil), (*kops.GossipConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_GossipConfig_To_kops_GossipConfig(a.(*GossipConfig), b.(*kops.GossipConfig), scope) }); err != nil { @@ -3317,6 +3327,15 @@ func autoConvert_v1alpha2_ContainerdConfig_To_kops_ContainerdConfig(in *Containe } else { out.NvidiaGPU = nil } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(kops.GVisorConfig) + if err := Convert_v1alpha2_GVisorConfig_To_kops_GVisorConfig(*in, *out, s); err != nil { + return err + } + } else { + out.GVisor = nil + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(kops.Runc) @@ -3376,6 +3395,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha2_ContainerdConfig(in *kops.Con } else { out.NvidiaGPU = nil } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(GVisorConfig) + if err := Convert_kops_GVisorConfig_To_v1alpha2_GVisorConfig(*in, *out, s); err != nil { + return err + } + } else { + out.GVisor = nil + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(Runc) @@ -4067,6 +4095,46 @@ func Convert_kops_GCPNetworkingSpec_To_v1alpha2_GCPNetworkingSpec(in *kops.GCPNe return autoConvert_kops_GCPNetworkingSpec_To_v1alpha2_GCPNetworkingSpec(in, out, s) } +func autoConvert_v1alpha2_GVisorConfig_To_kops_GVisorConfig(in *GVisorConfig, out *kops.GVisorConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Platform = in.Platform + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(kops.PackagesConfig) + if err := Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } + return nil +} + +// Convert_v1alpha2_GVisorConfig_To_kops_GVisorConfig is an autogenerated conversion function. +func Convert_v1alpha2_GVisorConfig_To_kops_GVisorConfig(in *GVisorConfig, out *kops.GVisorConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_GVisorConfig_To_kops_GVisorConfig(in, out, s) +} + +func autoConvert_kops_GVisorConfig_To_v1alpha2_GVisorConfig(in *kops.GVisorConfig, out *GVisorConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Platform = in.Platform + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + if err := Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } + return nil +} + +// Convert_kops_GVisorConfig_To_v1alpha2_GVisorConfig is an autogenerated conversion function. +func Convert_kops_GVisorConfig_To_v1alpha2_GVisorConfig(in *kops.GVisorConfig, out *GVisorConfig, s conversion.Scope) error { + return autoConvert_kops_GVisorConfig_To_v1alpha2_GVisorConfig(in, out, s) +} + func autoConvert_v1alpha2_GossipConfig_To_kops_GossipConfig(in *GossipConfig, out *kops.GossipConfig, s conversion.Scope) error { out.Protocol = in.Protocol out.Listen = in.Listen diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 6c1c4d9262b24..76cc28ce1e169 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1569,6 +1569,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) { *out = new(NvidiaGPUConfig) (*in).DeepCopyInto(*out) } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(GVisorConfig) + (*in).DeepCopyInto(*out) + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(Runc) @@ -2289,6 +2294,32 @@ func (in *GCPNetworkingSpec) DeepCopy() *GCPNetworkingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GVisorConfig) DeepCopyInto(out *GVisorConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GVisorConfig. +func (in *GVisorConfig) DeepCopy() *GVisorConfig { + if in == nil { + return nil + } + out := new(GVisorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GossipConfig) DeepCopyInto(out *GossipConfig) { *out = *in diff --git a/pkg/apis/kops/v1alpha3/containerdconfig.go b/pkg/apis/kops/v1alpha3/containerdconfig.go index f2b80f70f623a..02ddee7f0b9b7 100644 --- a/pkg/apis/kops/v1alpha3/containerdconfig.go +++ b/pkg/apis/kops/v1alpha3/containerdconfig.go @@ -48,6 +48,8 @@ type ContainerdConfig struct { Version *string `json:"version,omitempty"` // NvidiaGPU configures the Nvidia GPU runtime. NvidiaGPU *NvidiaGPUConfig `json:"nvidiaGPU,omitempty"` + // GVisor configures the gVisor (runsc) sandboxed runtime. + GVisor *GVisorConfig `json:"gvisor,omitempty"` // Runc configures the runc runtime. Runc *Runc `json:"runc,omitempty"` // SelinuxEnabled enables SELinux support @@ -99,3 +101,17 @@ type Runc struct { // Packages overrides the URL and hash for the packages. Packages *PackagesConfig `json:"packages,omitempty"` } + +// GVisorConfig configures the gVisor sandboxed container runtime. +// When enabled, kOps installs runsc and containerd-shim-runsc-v1, +// registers the "runsc" runtime handler in containerd, and deploys +// a Kubernetes RuntimeClass named "gvisor". +type GVisorConfig struct { + // Enabled determines if kOps will install the gVisor runtime. + Enabled *bool `json:"enabled,omitempty"` + // Platform is the gVisor execution platform: "systrap" (default, works + // everywhere including VMs) or "kvm" (bare-metal with KVM support). + Platform string `json:"platform,omitempty"` + // Packages overrides the URL and hash for the gVisor packages. + Packages *PackagesConfig `json:"packages,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index b8564cb7a0b3a..ac5adffa042a2 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -574,6 +574,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*GVisorConfig)(nil), (*kops.GVisorConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_GVisorConfig_To_kops_GVisorConfig(a.(*GVisorConfig), b.(*kops.GVisorConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.GVisorConfig)(nil), (*GVisorConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_GVisorConfig_To_v1alpha3_GVisorConfig(a.(*kops.GVisorConfig), b.(*GVisorConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*GossipConfig)(nil), (*kops.GossipConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_GossipConfig_To_kops_GossipConfig(a.(*GossipConfig), b.(*kops.GossipConfig), scope) }); err != nil { @@ -3591,6 +3601,15 @@ func autoConvert_v1alpha3_ContainerdConfig_To_kops_ContainerdConfig(in *Containe } else { out.NvidiaGPU = nil } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(kops.GVisorConfig) + if err := Convert_v1alpha3_GVisorConfig_To_kops_GVisorConfig(*in, *out, s); err != nil { + return err + } + } else { + out.GVisor = nil + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(kops.Runc) @@ -3650,6 +3669,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha3_ContainerdConfig(in *kops.Con } else { out.NvidiaGPU = nil } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(GVisorConfig) + if err := Convert_kops_GVisorConfig_To_v1alpha3_GVisorConfig(*in, *out, s); err != nil { + return err + } + } else { + out.GVisor = nil + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(Runc) @@ -4415,6 +4443,46 @@ func Convert_kops_GCPNetworkingSpec_To_v1alpha3_GCPNetworkingSpec(in *kops.GCPNe return autoConvert_kops_GCPNetworkingSpec_To_v1alpha3_GCPNetworkingSpec(in, out, s) } +func autoConvert_v1alpha3_GVisorConfig_To_kops_GVisorConfig(in *GVisorConfig, out *kops.GVisorConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Platform = in.Platform + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(kops.PackagesConfig) + if err := Convert_v1alpha3_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } + return nil +} + +// Convert_v1alpha3_GVisorConfig_To_kops_GVisorConfig is an autogenerated conversion function. +func Convert_v1alpha3_GVisorConfig_To_kops_GVisorConfig(in *GVisorConfig, out *kops.GVisorConfig, s conversion.Scope) error { + return autoConvert_v1alpha3_GVisorConfig_To_kops_GVisorConfig(in, out, s) +} + +func autoConvert_kops_GVisorConfig_To_v1alpha3_GVisorConfig(in *kops.GVisorConfig, out *GVisorConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Platform = in.Platform + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + if err := Convert_kops_PackagesConfig_To_v1alpha3_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } + return nil +} + +// Convert_kops_GVisorConfig_To_v1alpha3_GVisorConfig is an autogenerated conversion function. +func Convert_kops_GVisorConfig_To_v1alpha3_GVisorConfig(in *kops.GVisorConfig, out *GVisorConfig, s conversion.Scope) error { + return autoConvert_kops_GVisorConfig_To_v1alpha3_GVisorConfig(in, out, s) +} + func autoConvert_v1alpha3_GossipConfig_To_kops_GossipConfig(in *GossipConfig, out *kops.GossipConfig, s conversion.Scope) error { out.Protocol = in.Protocol out.Listen = in.Listen diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index b8a4aab0c129a..306ddadab9817 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -1480,6 +1480,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) { *out = new(NvidiaGPUConfig) (*in).DeepCopyInto(*out) } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(GVisorConfig) + (*in).DeepCopyInto(*out) + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(Runc) @@ -2246,6 +2251,32 @@ func (in *GCPNetworkingSpec) DeepCopy() *GCPNetworkingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GVisorConfig) DeepCopyInto(out *GVisorConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GVisorConfig. +func (in *GVisorConfig) DeepCopy() *GVisorConfig { + if in == nil { + return nil + } + out := new(GVisorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GossipConfig) DeepCopyInto(out *GossipConfig) { *out = *in diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 2b0f866b95baa..7826fef5cfc66 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1600,6 +1600,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) { *out = new(NvidiaGPUConfig) (*in).DeepCopyInto(*out) } + if in.GVisor != nil { + in, out := &in.GVisor, &out.GVisor + *out = new(GVisorConfig) + (*in).DeepCopyInto(*out) + } if in.Runc != nil { in, out := &in.Runc, &out.Runc *out = new(Runc) @@ -2409,6 +2414,32 @@ func (in *GCPNetworkingSpec) DeepCopy() *GCPNetworkingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GVisorConfig) DeepCopyInto(out *GVisorConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GVisorConfig. +func (in *GVisorConfig) DeepCopy() *GVisorConfig { + if in == nil { + return nil + } + out := new(GVisorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GossipConfig) DeepCopyInto(out *GossipConfig) { *out = *in diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go index 2baec53dd25d3..83d49c66276b4 100644 --- a/pkg/apis/nodeup/config.go +++ b/pkg/apis/nodeup/config.go @@ -105,6 +105,8 @@ type Config struct { DNSZone string `json:",omitempty"` // NvidiaGPU contains the configuration for nvidia NvidiaGPU *kops.NvidiaGPUConfig `json:",omitempty"` + // GVisor contains the configuration for gVisor sandboxed runtime + GVisor *kops.GVisorConfig `json:",omitempty"` // AWS-specific // DisableSecurityGroupIngress disables the Cloud Controller Manager's creation @@ -268,6 +270,10 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi config.NvidiaGPU = buildNvidiaConfig(cluster, instanceGroup) } + if (cluster.Spec.Containerd != nil && cluster.Spec.Containerd.GVisor != nil) || (instanceGroup.Spec.Containerd != nil && instanceGroup.Spec.Containerd.GVisor != nil) { + config.GVisor = buildGVisorConfig(cluster, instanceGroup) + } + config.KubeProxy = buildKubeProxy(cluster, instanceGroup) if cluster.Spec.NTP != nil && cluster.Spec.NTP.Managed != nil && !*cluster.Spec.NTP.Managed { @@ -452,6 +458,24 @@ func buildNvidiaConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) return config } +// buildGVisorConfig builds gVisor configuration for instance group +func buildGVisorConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) *kops.GVisorConfig { + config := &kops.GVisorConfig{} + if cluster.Spec.Containerd != nil && cluster.Spec.Containerd.GVisor != nil { + config = cluster.Spec.Containerd.GVisor + } + + if instanceGroup.Spec.Containerd != nil && instanceGroup.Spec.Containerd.GVisor != nil { + reflectutils.JSONMergeStruct(&config, instanceGroup.Spec.Containerd.GVisor) + } + + if config.Platform == "" { + config.Platform = kops.GVisorDefaultPlatform + } + + return config +} + // buildkubeProxy builds the kube-proxy configuration for an instance group. func buildKubeProxy(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) *kops.KubeProxyConfig { config := &kops.KubeProxyConfig{} diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go index 89e0fb14842a7..af3d6208bedf3 100644 --- a/pkg/model/components/containerd.go +++ b/pkg/model/components/containerd.go @@ -81,5 +81,11 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o *kops.Cluster) error { } } + if containerd.GVisor != nil && fi.ValueOf(containerd.GVisor.Enabled) { + if containerd.GVisor.Platform == "" { + containerd.GVisor.Platform = kops.GVisorDefaultPlatform + } + } + return nil } diff --git a/upup/models/cloudup/resources/addons/gvisor.addons.k8s.io/k8s-1.20.yaml.template b/upup/models/cloudup/resources/addons/gvisor.addons.k8s.io/k8s-1.20.yaml.template new file mode 100644 index 0000000000000..086f657863d82 --- /dev/null +++ b/upup/models/cloudup/resources/addons/gvisor.addons.k8s.io/k8s-1.20.yaml.template @@ -0,0 +1,8 @@ +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: gvisor +handler: runsc +scheduling: + nodeSelector: + kops.k8s.io/gvisor: "1" diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 7859f26ad163f..738960b064e94 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -615,6 +615,31 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } } + gvisor := b.Cluster.Spec.Containerd.GVisor + igGVisor := false + for _, ig := range b.KopsModelContext.InstanceGroups { + if ig.Spec.Containerd != nil && ig.Spec.Containerd.GVisor != nil && fi.ValueOf(ig.Spec.Containerd.GVisor.Enabled) { + igGVisor = true + break + } + } + + if gvisor != nil && fi.ValueOf(gvisor.Enabled) || igGVisor { + key := "gvisor.addons.k8s.io" + + { + location := key + "/k8s-1.20.yaml" + id := "k8s-1.20" + + addons.Add(&channelsapi.AddonSpec{ + Name: fi.PtrTo(key), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.PtrTo(location), + Id: id, + }) + } + } + if b.Cluster.Spec.CloudProvider.AWS != nil { if b.Cluster.Spec.CloudProvider.AWS.LoadBalancerController != nil && fi.ValueOf(b.Cluster.Spec.CloudProvider.AWS.LoadBalancerController.Enabled) { diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index da966cfb4da78..54995481b4845 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -229,6 +229,16 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, } } + // Label nodes that have gVisor enabled so the RuntimeClass nodeSelector works. + clusterGVisor := cluster.Spec.Containerd != nil && cluster.Spec.Containerd.GVisor != nil && fi.ValueOf(cluster.Spec.Containerd.GVisor.Enabled) + igGVisor := ig.Spec.Containerd != nil && ig.Spec.Containerd.GVisor != nil && fi.ValueOf(ig.Spec.Containerd.GVisor.Enabled) + if clusterGVisor || igGVisor { + if ig.Spec.NodeLabels == nil { + ig.Spec.NodeLabels = make(map[string]string) + } + ig.Spec.NodeLabels["kops.k8s.io/gvisor"] = "1" + } + if ig.Spec.Manager == "" { ig.Spec.Manager = kops.InstanceManagerCloudGroup } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index ecee73c3923a8..b63c482a2663b 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -310,6 +310,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.ManifestsBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.PackagesBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.NvidiaBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.GVisorBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.SecretBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.FirewallBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.SysctlBuilder{NodeupModelContext: modelContext})