runtime/nodeinstaller: add insecure platform variants

Author: msanft

internal/kuberesource/parts.go

@@ -28,6 +28,15 @@ func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, err
 
 	// Consists of the default VM memory, 70MiB for the Kata shim and 100MiB for qemu overhead.
 	memoryOverhead := platforms.DefaultMemoryInMebiBytes(platform) + 170
+	if platforms.IsInsecure(platform) && platforms.IsGPU(platform) {
+		// On insecure (non-CC) GPU platforms, iommufd VFIO passthrough pins guest
+		// memory and allocates IOMMU page tables that are charged to the pod's
+		// cgroup. On CC platforms, TDX manages this memory outside the cgroup.
+		// (in the kernel / TDX module memory)
+		// Add extra headroom to avoid OOM kills. 512MB should suffice for VMs up
+		// to ~256GB of memory, which is our current limit on TDX as well.
+		memoryOverhead += 512
+	}
 
 	r := RuntimeClass(runtimeHandler).
 		WithHandler(runtimeHandler).

internal/manifest/runtimehandler.go

@@ -33,16 +33,28 @@ func RuntimeHandler(platform platforms.Platform) (string, error) {
 // PlatformFromHandler extracts the platform from the runtime handler name.
 func PlatformFromHandler(handler string) (platforms.Platform, error) {
 	rest, found := strings.CutPrefix(handler, "contrast-cc-")
+	isInsecure := false
+	if !found {
+		rest, found = strings.CutPrefix(handler, "contrast-insecure-")
+		isInsecure = true
+	}
 	if !found {
 		return platforms.Unknown, fmt.Errorf("invalid handler name: %s", handler)
 	}
 
 	parts := strings.Split(rest, "-")
-	if len(parts) != 4 && len(parts) != 5 {
+	if len(parts) < 3 || len(parts) > 5 {
 		return platforms.Unknown, fmt.Errorf("invalid handler name: %s", handler)
 	}
 
 	rawPlatform := strings.Join(parts[:len(parts)-1], "-")
+	if isInsecure {
+		if before, ok := strings.CutSuffix(rawPlatform, "-gpu"); ok {
+			rawPlatform = before + "-insecure-gpu"
+		} else {
+			rawPlatform += "-insecure"
+		}
+	}
 
 	platform, err := platforms.FromString(rawPlatform)
 	if err != nil {

internal/platforms/platforms.go

@@ -24,11 +24,18 @@ const (
 	MetalQEMUSNPGPU
 	// MetalQEMUTDXGPU is the generic platform for bare-metal TDX deployments with GPU passthrough.
 	MetalQEMUTDXGPU
+	// MetalQEMUInsecure is the platform for bare-metal deployments with a non-CC runtime class.
+	MetalQEMUInsecure
+	// MetalQEMUInsecureGPU is the platform for bare-metal deployments with GPU passthrough and a non-CC runtime class.
+	MetalQEMUInsecureGPU
 )
 
 // All returns a list of all available platforms.
 func All() []Platform {
-	return []Platform{MetalQEMUSNP, MetalQEMUTDX, MetalQEMUSNPGPU, MetalQEMUTDXGPU}
+	return []Platform{
+		MetalQEMUSNP, MetalQEMUTDX, MetalQEMUSNPGPU, MetalQEMUTDXGPU,
+		MetalQEMUInsecure, MetalQEMUInsecureGPU,
+	}
 }
 
 // AllStrings returns a list of all available platforms as strings.
@@ -51,11 +58,28 @@ func (p Platform) String() string {
 		return "Metal-QEMU-TDX"
 	case MetalQEMUTDXGPU:
 		return "Metal-QEMU-TDX-GPU"
+	case MetalQEMUInsecure:
+		return "Metal-QEMU-Insecure"
+	case MetalQEMUInsecureGPU:
+		return "Metal-QEMU-Insecure-GPU"
 	default:
 		return "Unknown"
 	}
 }
 
+// InsecureVariant returns the insecure (non-CC) variant of the
+// platform, or Unknown if there is no such variant.
+func (p Platform) InsecureVariant() Platform {
+	switch p {
+	case MetalQEMUSNP, MetalQEMUTDX:
+		return MetalQEMUInsecure
+	case MetalQEMUSNPGPU, MetalQEMUTDXGPU:
+		return MetalQEMUInsecureGPU
+	default:
+		return Unknown
+	}
+}
+
 // MarshalJSON marshals a Platform type to a JSON string.
 func (p Platform) MarshalJSON() ([]byte, error) {
 	return fmt.Appendf(nil, `"%s"`, p.String()), nil
@@ -99,6 +123,10 @@ func FromString(s string) (Platform, error) {
 		return MetalQEMUTDX, nil
 	case "metal-qemu-tdx-gpu":
 		return MetalQEMUTDXGPU, nil
+	case "metal-qemu-insecure":
+		return MetalQEMUInsecure, nil
+	case "metal-qemu-insecure-gpu":
+		return MetalQEMUInsecureGPU, nil
 	default:
 		return Unknown, fmt.Errorf("unknown platform: %s", s)
 	}
@@ -125,6 +153,10 @@ func FromRuntimeClassString(s string) (Platform, error) {
 		return MetalQEMUTDXGPU, nil
 	case strings.HasPrefix(s, "contrast-cc-metal-qemu-tdx"):
 		return MetalQEMUTDX, nil
+	case strings.HasPrefix(s, "contrast-insecure-metal-qemu-gpu"):
+		return MetalQEMUInsecureGPU, nil
+	case strings.HasPrefix(s, "contrast-insecure-metal-qemu"):
+		return MetalQEMUInsecure, nil
 	default:
 		return Unknown, fmt.Errorf("unknown platform: %s", s)
 	}
@@ -133,7 +165,7 @@ func FromRuntimeClassString(s string) (Platform, error) {
 // DefaultMemoryInMebiBytes returns the desired VM overhead for the given platform.
 func DefaultMemoryInMebiBytes(p Platform) int {
 	switch p {
-	case MetalQEMUSNPGPU, MetalQEMUTDXGPU:
+	case MetalQEMUSNPGPU, MetalQEMUTDXGPU, MetalQEMUInsecureGPU:
 		// Guest components contribute around 600MiB with GPU enabled.
 		return 1024
 	default:
@@ -144,6 +176,16 @@ func DefaultMemoryInMebiBytes(p Platform) int {
 	}
 }
 
+// IsInsecure returns true if the platform is an insecure (non-CC) platform.
+func IsInsecure(p Platform) bool {
+	switch p {
+	case MetalQEMUInsecure, MetalQEMUInsecureGPU:
+		return true
+	default:
+		return false
+	}
+}
+
 // IsSNP returns true if the platform is a SEV-SNP platform.
 func IsSNP(p Platform) bool {
 	switch p {
@@ -167,7 +209,7 @@ func IsTDX(p Platform) bool {
 // IsGPU returns true if the platform supports GPUs.
 func IsGPU(p Platform) bool {
 	switch p {
-	case MetalQEMUSNPGPU, MetalQEMUTDXGPU:
+	case MetalQEMUSNPGPU, MetalQEMUTDXGPU, MetalQEMUInsecureGPU:
 		return true
 	default:
 		return false
@@ -177,7 +219,8 @@ func IsGPU(p Platform) bool {
 // IsQEMU returns true if the platform uses QEMU as the hypervisor.
 func IsQEMU(p Platform) bool {
 	switch p {
-	case MetalQEMUSNP, MetalQEMUSNPGPU, MetalQEMUTDX, MetalQEMUTDXGPU:
+	case MetalQEMUSNP, MetalQEMUSNPGPU, MetalQEMUTDX, MetalQEMUTDXGPU,
+		MetalQEMUInsecure, MetalQEMUInsecureGPU:
 		return true
 	default:
 		return false
@@ -191,6 +234,8 @@ func (p Platform) WithGPU() Platform {
 		return MetalQEMUSNPGPU
 	case MetalQEMUTDX, MetalQEMUTDXGPU:
 		return MetalQEMUTDXGPU
+	case MetalQEMUInsecure, MetalQEMUInsecureGPU:
+		return MetalQEMUInsecureGPU
 	default:
 		return Unknown
 	}

justfile

@@ -58,10 +58,10 @@ node-installer platform=default_platform:
     #!/usr/bin/env bash
     set -euo pipefail
     case {{ platform }} in
-        "Metal-QEMU-SNP"|"Metal-QEMU-TDX")
+        "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-Insecure")
             just push "node-installer-kata"
         ;;
-        "Metal-QEMU-SNP-GPU"|"Metal-QEMU-TDX-GPU")
+        "Metal-QEMU-SNP-GPU"|"Metal-QEMU-TDX-GPU"|"Metal-QEMU-Insecure-GPU")
             just push "node-installer-kata-gpu"
         ;;
         *)

nodeinstaller/internal/containerdconfig/config.go

@@ -144,12 +144,12 @@ func ContrastRuntime(baseDir string, platform platforms.Platform) (Runtime, erro
 		PrivilegedWithoutHostDevices: true,
 	}
 
-	switch {
-	case platforms.IsTDX(platform):
+	switch platform {
+	case platforms.MetalQEMUTDX, platforms.MetalQEMUTDXGPU:
 		cfg.Options = map[string]any{
 			"ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-tdx.toml"),
 		}
-	case platforms.IsSNP(platform):
+	case platforms.MetalQEMUSNP, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUInsecure, platforms.MetalQEMUInsecureGPU:
 		cfg.Options = map[string]any{
 			"ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-snp.toml"),
 		}

nodeinstaller/internal/kataconfig/config.go

@@ -39,30 +39,39 @@ func KataRuntimeConfig(
 ) (*Config, error) {
 	var customContrastAnnotations []string
 	var config Config
-	switch {
-	case platforms.IsTDX(platform):
+	switch platform {
+	case platforms.MetalQEMUTDX, platforms.MetalQEMUTDXGPU:
 		if err := toml.Unmarshal([]byte(kataBareMetalQEMUTDXBaseConfig), &config); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
 		}
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "tdx", "share", "OVMF.fd")
 		// We set up dm_verity in the system NixOS config.
 		// Doing so again here prevents VM boots.
 		config.Hypervisor["qemu"]["kernel_verity_params"] = ""
-	case platforms.IsSNP(platform):
+	case platforms.MetalQEMUSNP, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUInsecure, platforms.MetalQEMUInsecureGPU:
 		if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
 		}
 
-		for _, productLine := range []string{"_Milan", "_Genoa"} {
-			for _, annotationType := range []string{"snp_id_block", "snp_id_auth", "snp_guest_policy"} {
-				customContrastAnnotations = append(customContrastAnnotations, annotationType+productLine)
+		if !platforms.IsInsecure(platform) {
+			for _, productLine := range []string{"_Milan", "_Genoa"} {
+				for _, annotationType := range []string{"snp_id_block", "snp_id_auth", "snp_guest_policy"} {
+					customContrastAnnotations = append(customContrastAnnotations, annotationType+productLine)
+				}
 			}
 		}
 
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "snp", "share", "OVMF.fd")
 	default:
 		return nil, fmt.Errorf("unsupported platform: %s", platform)
 	}
+	// Disable confidential computing features for insecure platforms.
+	if platforms.IsInsecure(platform) {
+		config.Hypervisor["qemu"]["confidential_guest"] = false
+		if platforms.IsSNP(platform) || platform == platforms.MetalQEMUInsecure || platform == platforms.MetalQEMUInsecureGPU {
+			config.Hypervisor["qemu"]["sev_snp_guest"] = false
+		}
+	}
 	if debug {
 		config.Agent["kata"]["enable_debug"] = true
 		config.Agent["kata"]["debug_console_enabled"] = true

nodeinstaller/internal/kataconfig/config_test.go

@@ -22,6 +22,10 @@ var (
 	expectedConfMetalQEMUSNPGPU []byte
 	//go:embed testdata/expected-configuration-qemu-tdx-gpu.toml
 	expectedConfMetalQEMUTDXGPU []byte
+	//go:embed testdata/expected-configuration-qemu-insecure.toml
+	expectedConfMetalQEMUInsecure []byte
+	//go:embed testdata/expected-configuration-qemu-insecure-gpu.toml
+	expectedConfMetalQEMUInsecureGPU []byte
 )
 
 func TestKataRuntimeConfig(t *testing.T) {
@@ -45,6 +49,14 @@ func TestKataRuntimeConfig(t *testing.T) {
 			changeSnpFields: false,
 			want:            string(expectedConfMetalQEMUTDXGPU),
 		},
+		platforms.MetalQEMUInsecure: {
+			changeSnpFields: true,
+			want:            string(expectedConfMetalQEMUInsecure),
+		},
+		platforms.MetalQEMUInsecureGPU: {
+			changeSnpFields: true,
+			want:            string(expectedConfMetalQEMUInsecureGPU),
+		},
 	}
 	for platform, tc := range testCases {
 		t.Run(platform.String(), func(t *testing.T) {

nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-insecure-gpu.toml

@@ -0,0 +1,117 @@
+[Hypervisor]
+[Hypervisor.qemu]
+block_device_aio = 'threads'
+block_device_cache_direct = false
+block_device_cache_noflush = false
+block_device_cache_set = false
+block_device_driver = 'virtio-scsi'
+cold_plug_vfio = 'root-port'
+confidential_guest = false
+contrast_imagepuller_config = ''
+cpu_features = 'pmu=off'
+default_bridges = 1
+default_maxmemory = 0
+default_maxvcpus = 0
+default_memory = 1024
+default_vcpus = 1
+disable_block_device_use = true
+disable_guest_selinux = true
+disable_image_nvdimm = true
+disable_nesting_checks = true
+disable_selinux = false
+disable_vhost_net = false
+enable_annotations = ['cc_init_data']
+enable_debug = false
+enable_guest_swap = false
+enable_hugepages = false
+enable_iommu = false
+enable_iommu_platform = false
+enable_iothreads = false
+enable_mem_prealloc = false
+enable_numa = false
+enable_vhost_user_store = false
+enable_virtio_mem = false
+entropy_source = '/dev/urandom'
+file_mem_backend = ''
+firmware = '/snp/share/OVMF.fd'
+firmware_volume = ''
+guest_hook_path = ''
+guest_memory_dump_paging = false
+guest_memory_dump_path = ''
+image = '/share/kata-containers.img'
+indep_iothreads = 0
+initrd = '/share/kata-initrd.zst'
+kernel = '/share/kata-kernel'
+kernel_params = ''
+machine_accelerators = ''
+machine_type = 'q35'
+memory_offset = 0
+memory_slots = 10
+msize_9p = 8192
+numa_mapping = []
+path = '/bin/qemu-system-x86_64'
+pcie_root_port = 0
+pflashes = []
+reclaim_guest_freed_memory = false
+rootfs_type = 'erofs'
+rootless = false
+rx_rate_limiter_max_rate = 0
+seccompsandbox = ''
+sev_snp_guest = false
+shared_fs = 'none'
+snp_guest_policy = 196608
+snp_id_auth = ''
+snp_id_block = ''
+tx_rate_limiter_max_rate = 0
+use_legacy_serial = false
+valid_entropy_sources = ['/dev/urandom', '/dev/random', '']
+valid_file_mem_backends = ['']
+valid_hypervisor_paths = ['/bin/qemu-system-x86_64']
+valid_vhost_user_store_paths = ['/var/run/kata-containers/vhost-user']
+valid_virtio_fs_daemon_paths = ['/opt/kata/libexec/virtiofsd']
+vhost_user_reconnect_timeout_sec = 0
+vhost_user_store_path = '/var/run/kata-containers/vhost-user'
+virtio_fs_cache = 'auto'
+virtio_fs_cache_size = 0
+virtio_fs_daemon = '/opt/kata/libexec/virtiofsd'
+virtio_fs_extra_args = ['--thread-pool-size=1', '--announce-submounts']
+virtio_fs_queue_size = 1024
+
+[Agent]
+[Agent.kata]
+debug_console_enabled = false
+dial_timeout = 600
+enable_debug = false
+enable_tracing = false
+kernel_modules = []
+
+[Factory]
+enable_template = false
+template_path = '/run/vc/vm/template'
+vm_cache_endpoint = '/var/run/kata-containers/cache.sock'
+vm_cache_number = 0
+
+[Runtime]
+create_container_timeout = 600
+dan_conf = '/run/kata-containers/dans'
+disable_guest_empty_dir = false
+disable_guest_seccomp = true
+disable_new_netns = false
+emptydir_mode = 'shared-fs'
+enable_debug = false
+enable_pprof = false
+enable_tracing = false
+enable_vcpus_pinning = false
+experimental = []
+experimental_force_guest_pull = true
+guest_selinux_label = ''
+internetworking_model = 'tcfilter'
+jaeger_endpoint = ''
+jaeger_password = ''
+jaeger_user = ''
+kubelet_root_dir = '/var/lib/kubelet'
+pod_resource_api_sock = '/var/lib/kubelet/pod-resources/kubelet.sock'
+sandbox_bind_mounts = []
+sandbox_cgroup_only = true
+static_sandbox_resource_mgmt = true
+vfio_mode = 'guest-kernel'