ci: Introduce tests for VM template factory

Add k8s-vm-templating-test.bats which exercises pod create
with the factory initialized on the target node.

Signed-off-by: Cameron Baird <cameronbaird@microsoft.com>

Author: Camelron

.github/workflows/run-k8s-tests-on-free-runner.yaml

@@ -41,10 +41,12 @@ jobs:
       matrix:
         environment: [
           { vmm: clh, containerd_version: latest },
+          { vmm: clh, containerd_version: latest, snapshotter: erofs, erofs_mode: disk, erofs_merge_mode: unmerged },
           { vmm: clh, containerd_version: minimum },
           { vmm: dragonball, containerd_version: latest },
           { vmm: dragonball, containerd_version: minimum },
           { vmm: qemu, containerd_version: latest },
+          { vmm: qemu, containerd_version: latest, snapshotter: erofs, erofs_mode: disk, erofs_merge_mode: unmerged },
           { vmm: qemu, containerd_version: minimum },
           { vmm: qemu-runtime-rs, containerd_version: latest },
           { vmm: qemu-runtime-rs, containerd_version: minimum },
@@ -68,6 +70,9 @@ jobs:
       K8S_TEST_HOST_TYPE: baremetal-no-attestation
       CONTAINER_ENGINE: containerd
       CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }}
+      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
+      EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.erofs_mode }}
+      EROFS_MERGE_MODE: ${{ matrix.environment.erofs_merge_mode }}
       GH_TOKEN: ${{ github.token }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

src/runtime/virtcontainers/clh.go

@@ -816,15 +816,20 @@ func (clh *cloudHypervisor) shouldRestoreFromTemplate() bool {
 	return true
 }
 
-// copyFile copies a file from src to dst
+// copyFile copies a file from src to dst, preserving the source file's permissions.
 func (clh *cloudHypervisor) copyFile(src, dst string) error {
 	srcFile, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer srcFile.Close()
 
-	dstFile, err := os.Create(dst)
+	srcInfo, err := srcFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, srcInfo.Mode())
 	if err != nil {
 		return err
 	}
@@ -874,7 +879,7 @@ func (clh *cloudHypervisor) updateVsockSocketPath(configPath, vmID string) error
 		return err
 	}
 
-	return os.WriteFile(configPath, updatedConfig, 0644)
+	return os.WriteFile(configPath, updatedConfig, 0600)
 }
 
 // setupInitdata prepares and attaches the initdata disk if present.
@@ -1572,7 +1577,7 @@ func (clh *cloudHypervisor) SaveVM() error {
 			return err
 		}
 
-		if err := os.WriteFile(snapshotConfigPath, modifiedConfig, 0644); err != nil {
+		if err := os.WriteFile(snapshotConfigPath, modifiedConfig, 0600); err != nil {
 			clh.Logger().WithError(err).Error("Failed to write modified snapshot config")
 			return err
 		}

src/runtime/virtcontainers/clh_test.go

@@ -457,7 +457,8 @@ func TestCloudHypervisorCleanupVM(t *testing.T) {
 	assert.NoError(err, "persist.GetDriver() unexpected error")
 
 	dir := filepath.Join(store.RunVMStoragePath(), clh.id)
-	os.MkdirAll(dir, os.ModePerm)
+	err = os.MkdirAll(dir, os.ModePerm)
+	assert.NoError(err, "failed to create dir %s", dir)
 
 	err = clh.cleanupVM(false)
 	assert.NoError(err, "persist.GetDriver() unexpected error")
@@ -566,7 +567,8 @@ func TestClhRestoreVM(t *testing.T) {
 	assert.Contains(err.Error(), filepath.Join(clhConfig.VMStorePath, "state.json"))
 
 	// Now create the VM snapshot files and call restoreVM again.
-	os.MkdirAll(clhConfig.VMStorePath, os.ModePerm)
+	err = os.MkdirAll(clhConfig.VMStorePath, os.ModePerm)
+	assert.NoError(err, "failed to create dir %s", clhConfig.VMStorePath)
 	stateFile := filepath.Join(clhConfig.VMStorePath, "state.json")
 	configFile := filepath.Join(clhConfig.VMStorePath, "config.json")
 	err = os.WriteFile(stateFile, []byte("{}"), 0o600)

src/runtime/virtcontainers/factory/factory_linux.go

@@ -82,7 +82,6 @@ func resetHypervisorConfig(config *vc.VMConfig) {
 	config.HypervisorConfig.RunStorePath = ""
 	config.HypervisorConfig.SandboxName = ""
 	config.HypervisorConfig.SandboxNamespace = ""
-	config.HypervisorConfig.DefaultMaxVCPUs = 0
 }
 
 // It's important that baseConfig and newConfig are passed by value!

tests/gha-run-k8s-common.sh

@@ -43,6 +43,7 @@ TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-}"
 CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
 SNAPSHOTTER="${SNAPSHOTTER:-}"
 EROFS_SNAPSHOTTER_MODE="${EROFS_SNAPSHOTTER_MODE:-}"
+EROFS_MERGE_MODE="${EROFS_MERGE_MODE:-}"
 
 # Wait for the Kubernetes API to recover after kata-deploy uninstall, then
 # retry the uninstall to purge any stale helm release state. On k3s/rke2,
@@ -851,6 +852,26 @@ function helm_helper() {
 				yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}"
 		fi
 
+		# EROFS merge mode ("merged" default, or "unmerged"). This is orthogonal
+		# to EROFS_SNAPSHOTTER_MODE (which controls default_size): it controls
+		# whether containerd merges layers into a single fsmeta.erofs (merged,
+		# runtime-rs only) or keeps per-layer layer.erofs (unmerged, required by
+		# the Go runtime).
+		if [[ -n "${EROFS_MERGE_MODE}" ]]; then
+			if [[ "${SNAPSHOTTER}" != "erofs" ]]; then
+				die "EROFS_MERGE_MODE is only supported with SNAPSHOTTER=erofs"
+			fi
+
+			case "${EROFS_MERGE_MODE}" in
+				merged|unmerged) ;;
+				*)
+					die "Unsupported EROFS_MERGE_MODE: ${EROFS_MERGE_MODE}"
+					;;
+			esac
+
+			yq -i ".snapshotter.erofsMergeMode = \"${EROFS_MERGE_MODE}\"" "${values_yaml}"
+		fi
+
 		if [[ -z "${HELM_SHIMS}" ]]; then
 			die "A list of shims is expected but none was provided"
 		fi

tests/integration/kubernetes/k8s-vm-templating.bats

@@ -0,0 +1,104 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2024 Kata Containers
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Tests for Kata VM templating (factory) functionality in Kubernetes integration mode
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/confidential_common.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+# Returns 0 if the current environment supports VM templating, non-zero
+# otherwise. VM templating is only supported on non-confidential clh/qemu
+# hypervisors, and because it uses shared_fs="none" it also requires a
+# block-device-based snapshotter (blockfile or erofs).
+vm_templating_supported() {
+	[[ "${KATA_HYPERVISOR}" == "clh" || "${KATA_HYPERVISOR}" == "qemu" ]] || return 1
+	is_confidential_runtime_class && return 1
+	[[ "${SNAPSHOTTER:-}" =~ ^(blockfile|erofs)$ ]] || return 1
+	return 0
+}
+
+setup() {
+	if ! vm_templating_supported; then
+		skip "VM templating requires a non-confidential clh/qemu hypervisor and a blockfile/erofs snapshotter (KATA_HYPERVISOR=${KATA_HYPERVISOR}, SNAPSHOTTER=${SNAPSHOTTER:-unset})"
+	fi
+
+	setup_common || die "setup_common failed"
+
+	# Build a Kata runtime config drop-in that enables VM templating and
+	# disables shared_fs (incompatible with templating).
+	local runtime_config_dropin_file="${BATS_TEST_TMPDIR}/99-k8s-vm-templating.toml"
+	cat > "${runtime_config_dropin_file}" <<DROPIN
+[hypervisor.${KATA_HYPERVISOR}]
+shared_fs = "none"
+default_vcpus = 1
+default_memory = 512
+
+[factory]
+enable_template = true
+template_path = "/run/vc/vm/template"
+DROPIN
+
+	# Install the drop-in on the node selected by setup_common and record the
+	# remote path so teardown can remove it.
+	dropin_path="$(set_kata_runtime_config_dropin_file "$node" "${runtime_config_dropin_file}")" \
+		|| die "Failed to install Kata runtime config drop-in on node $node"
+}
+
+@test "Pod can be created with a templated VM" {
+	# Initialize the VM template on the target node. Use the absolute path:
+	# kata-deploy installs kata-runtime under /opt/kata/bin, which is not on the
+	# node's sudo PATH.
+	exec_host "$node" "sudo /opt/kata/bin/kata-runtime factory init"
+
+	# The factory init above must have created the template directory. exec_host
+	# pipes the remote output through `tr`, so the pipeline's exit status is not
+	# the remote command's; assert on the output instead.
+	exec_host "$node" "test -d /run/vc/vm/template && echo present" | grep -q present
+
+	pod_name="test-templated-pod"
+	ctr_name="test-container"
+
+	pod_config=$(mktemp --tmpdir pod_config.XXXXXX.yaml)
+	cp "$pod_config_dir/busybox-template.yaml" "$pod_config"
+
+	sed -i "s/POD_NAME/$pod_name/" "$pod_config"
+	sed -i "s/CTR_NAME/$ctr_name/" "$pod_config"
+
+	kubectl create -f "${pod_config}"
+	kubectl wait --for=condition=Ready --timeout="$timeout" "pod/${pod_name}"
+
+	grep_pod_exec_output "${pod_name}" "Hello from templated VM" sh -c "echo 'Hello from templated VM'"
+
+	# Confirm the pod's VM was actually spawned from the factory/template
+	# rather than booted normally. A normal VM stores its state directly under
+	# /run/vc/vm/<sandbox-id>/ (a real directory), whereas a factory-spawned VM
+	# is created under a generated UUID directory and /run/vc/vm/<sandbox-id> is
+	# a symlink pointing at it (see assignSandbox() in
+	# src/runtime/virtcontainers/vm.go). With templating enabled, the factory is
+	# the template factory, so the symlink is our signal the template was used.
+	local sandbox_id
+	sandbox_id="$(exec_host "$node" \
+		"crictl --runtime-endpoint unix:///run/containerd/containerd.sock pods --name ${pod_name} --state Ready -q | head -1")"
+
+	exec_host "$node" "test -L /run/vc/vm/${sandbox_id} && echo symlink" | grep -q symlink
+}
+
+teardown() {
+	vm_templating_supported || return 0
+
+	rm -f "${pod_config:-}"
+
+	# Destroy the VM template and remove the config drop-in on the target node.
+	exec_host "$node" "sudo /opt/kata/bin/kata-runtime factory destroy" \
+		|| echo "Warning: Failed to destroy VM template on node $node"
+
+	remove_kata_runtime_config_dropin_file "$node" "${dropin_path:-}" \
+		|| echo "Warning: Failed to remove Kata runtime config drop-in on node $node"
+
+	teardown_common "${node:-}" "${node_start_time:-}"
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -104,6 +104,7 @@ else
 		"k8s-security-context.bats" \
 		"k8s-shared-volume.bats" \
 		"k8s-volume.bats" \
+		"k8s-vm-templating.bats" \
 		"k8s-nginx-connectivity.bats" \
 	)
 

tools/packaging/kata-deploy/binary/src/artifacts/snapshotters.rs

@@ -15,36 +15,50 @@ use std::path::Path;
 pub async fn configure_erofs_snapshotter(config: &Config, configuration_file: &Path) -> Result<()> {
     info!("Configuring erofs-snapshotter");
 
+    // "unmerged" mode keeps each image layer as its own per-layer `layer.erofs`
+    // (containerd's default, non-fsmerged layout), which is the only layout the
+    // Go runtime can consume. In the default "merged" mode we force containerd
+    // to merge layers into a single `fsmeta.erofs`, which is runtime-rs only.
+    let unmerged = config.erofs_merge_mode.as_deref() == Some("unmerged");
+
     // The Go runtime does not support fsmerged EROFS (fsmeta.erofs).
     // If the snapshotter handler mapping explicitly pairs a Go shim with
-    // erofs, that is a hard misconfiguration — bail out so the operator
-    // fixes the mapping instead of hitting cryptic runtime errors later.
-    if let Some(mapping) = config.snapshotter_handler_mapping_for_arch.as_ref() {
-        let mut go_shims_on_erofs = Vec::new();
-        for entry in mapping.split(',') {
-            let parts: Vec<&str> = entry.split(':').collect();
-            if parts.len() == 2 && parts[1] == "erofs" && !utils::is_rust_shim(parts[0]) {
-                go_shims_on_erofs.push(parts[0].to_string());
+    // erofs in the (default) merged mode, that is a hard misconfiguration —
+    // bail out so the operator fixes the mapping instead of hitting cryptic
+    // runtime errors later. In "unmerged" mode the Go runtime is supported, so
+    // skip this guard.
+    if !unmerged {
+        if let Some(mapping) = config.snapshotter_handler_mapping_for_arch.as_ref() {
+            let mut go_shims_on_erofs = Vec::new();
+            for entry in mapping.split(',') {
+                let parts: Vec<&str> = entry.split(':').collect();
+                if parts.len() == 2 && parts[1] == "erofs" && !utils::is_rust_shim(parts[0]) {
+                    go_shims_on_erofs.push(parts[0].to_string());
+                }
             }
-        }
-        if !go_shims_on_erofs.is_empty() {
-            warn!("##########################################################################");
-            warn!("#                                                                        #");
-            warn!("#  Go runtime shim(s) mapped to the erofs snapshotter:                   #");
-            for s in &go_shims_on_erofs {
-                warn!("#    - {:<64} #", s);
+            if !go_shims_on_erofs.is_empty() {
+                warn!("##########################################################################");
+                warn!("#                                                                        #");
+                warn!("#  Go runtime shim(s) mapped to the erofs snapshotter:                   #");
+                for s in &go_shims_on_erofs {
+                    warn!("#    - {:<64} #", s);
+                }
+                warn!("#                                                                        #");
+                warn!(
+                    "#  The Go runtime does NOT support fsmerged EROFS (fsmeta.erofs).         #"
+                );
+                warn!("#  Only runtime-rs shims are supported with merged erofs. Set            #");
+                warn!("#  EROFS_MERGE_MODE=unmerged to use the Go runtime with erofs.           #");
+                warn!("#                                                                        #");
+                warn!("##########################################################################");
+                return Err(anyhow::anyhow!(
+                    "erofs snapshotter: Go runtime shim(s) [{}] cannot be mapped to merged erofs. \
+                     The Go runtime does not support fsmerged EROFS. \
+                     Set EROFS_MERGE_MODE=unmerged, remove these shims from \
+                     SNAPSHOTTER_HANDLER_MAPPING, or switch them to runtime-rs.",
+                    go_shims_on_erofs.join(", ")
+                ));
             }
-            warn!("#                                                                        #");
-            warn!("#  The Go runtime does NOT support fsmerged EROFS (fsmeta.erofs).         #");
-            warn!("#  Only runtime-rs shims are supported with the erofs snapshotter.        #");
-            warn!("#                                                                        #");
-            warn!("##########################################################################");
-            return Err(anyhow::anyhow!(
-                "erofs snapshotter: Go runtime shim(s) [{}] cannot be mapped to erofs. \
-                 The Go runtime does not support fsmerged EROFS. \
-                 Remove these shims from SNAPSHOTTER_HANDLER_MAPPING or switch them to runtime-rs.",
-                go_shims_on_erofs.join(", ")
-            ));
         }
     }
 
@@ -88,11 +102,27 @@ pub async fn configure_erofs_snapshotter(config: &Config, configuration_file: &P
         ".plugins.\"io.containerd.snapshotter.v1.erofs\".default_size",
         "\"10G\"",
     )?;
-    toml_utils::set_toml_value(
-        configuration_file,
-        ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers",
-        "0",
-    )?;
+    // In the default "merged" mode, force containerd to merge all layers into a
+    // single fsmeta.erofs (max_unmerged_layers = 0). In "unmerged" mode we delete
+    // any previously-written value so each layer stays a separate layer.erofs,
+    // which the Go runtime requires.
+    //
+    // Because kata-deploy edits the containerd config in place, switching from
+    // merged to unmerged must actively remove the old `max_unmerged_layers = 0`
+    // left behind by a previous install. Otherwise the stale `0` would keep
+    // forcing the merged layout and break Go-runtime compatibility.
+    if !unmerged {
+        toml_utils::set_toml_value(
+            configuration_file,
+            ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers",
+            "0",
+        )?;
+    } else {
+        toml_utils::delete_toml_value(
+            configuration_file,
+            ".plugins.\"io.containerd.snapshotter.v1.erofs\".max_unmerged_layers",
+        )?;
+    }
 
     Ok(())
 }