feat(vmm): add swtpm key provider for no-TEE dev VMs

Author: kvinwang

CONTRIBUTING.md

@@ -62,6 +62,12 @@ dstack deploy ./docker-compose.yml \
   --no-tee
 ```
 
+To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM
+host and select `key_provider=tpm` in the VMM console (or pass `--key-provider
+tpm` in tooling that exposes the compose option). The VMM keeps the software
+TPM state in the VM work directory so that seal/unseal survives guest restarts.
+This development provider is available only to no-TEE development images.
+
 The simulator package is installed only in development images. This mode
 provides no hardware isolation and must never be used with production
 workloads or secrets. Real quote generation, hardware isolation, and KMS

docs/security/cvm-boundaries.md

@@ -44,7 +44,7 @@ This is the main configuration file for the application in JSON format:
 | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup |
 | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. |
 | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). |
-| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". `"tpm"` is only supported on platforms whose TPM is part of the platform trust model (GCP vTPM, AWS EC2 NitroTPM); on other platforms guest setup fails closed, since sealing to a host-provided software TPM (e.g. swtpm under bare QEMU) offers no protection against the host. |
+| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack development platform can use VMM-managed swtpm to exercise seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. |
 
 
 The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation.

dstack/tpm-attest/src/lib.rs

@@ -48,7 +48,7 @@ const AWS_NITRO_PCRS: [u32; 5] = [4, 7, 8, 12, 14];
 
 pub fn dstack_pcr_policy_for_platform(platform: Platform) -> Result<PcrSelection> {
     match platform {
-        Platform::Gcp => Ok(dstack_pcr_policy()),
+        Platform::Dstack | Platform::Gcp => Ok(dstack_pcr_policy()),
         Platform::AwsEc2 => Ok(PcrSelection::sha384(&AWS_NITRO_PCRS)),
         _ => bail!("TPM local key provider is not supported on {platform:?}"),
     }
@@ -367,6 +367,16 @@ mod tests {
         let policy = dstack_pcr_policy_for_platform(Platform::AwsEc2).unwrap();
         assert_eq!(policy.to_arg(), "sha384:4,7,8,12,14");
     }
+
+    #[test]
+    fn dstack_platform_uses_development_tpm_policy() {
+        assert_eq!(
+            dstack_pcr_policy_for_platform(Platform::Dstack)
+                .unwrap()
+                .to_arg(),
+            "sha256:0,2,14"
+        );
+    }
 }
 
 mod gcp_ak;

dstack/vmm/src/app.rs

@@ -399,6 +399,7 @@ impl App {
             }
             for process in processes {
                 if let Err(err) = self.supervisor.deploy(&process).await {
+                    self.stop_dependent_processes(id).await;
                     if let Err(clear_err) = work_dir.clear_runtime_networks() {
                         warn!(
                             id,
@@ -430,8 +431,24 @@ impl App {
     pub async fn stop_vm(&self, id: &str) -> Result<()> {
         self.set_started(id, false)?;
         self.cleanup_port_forward(id).await;
-        self.supervisor.stop(id).await?;
-        Ok(())
+        let result = self.supervisor.stop(id).await;
+        self.stop_dependent_processes(id).await;
+        result
+    }
+
+    async fn stop_dependent_processes(&self, id: &str) {
+        for process in self.supervisor.list().await.unwrap_or_default() {
+            let note =
+                serde_json::from_str::<ProcessAnnotation>(&process.config.note).unwrap_or_default();
+            if note.live_for.as_deref() == Some(id) {
+                if let Err(err) = self.supervisor.stop(&process.config.id).await {
+                    debug!(
+                        process_id = process.config.id,
+                        "failed to stop dependent process: {err:?}"
+                    );
+                }
+            }
+        }
     }
 
     pub async fn remove_vm(&self, id: &str) -> Result<()> {
@@ -475,6 +492,7 @@ impl App {
         if let Err(err) = self.supervisor.stop(id).await {
             debug!("supervisor.stop({id}) during removal: {err:?}");
         }
+        self.stop_dependent_processes(id).await;
 
         // Poll until the process is no longer running, then remove it.
         // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely.
@@ -509,6 +527,31 @@ impl App {
             }
         }
 
+        // Auxiliary processes (currently swtpm) are owned by the VM through
+        // ProcessAnnotation::live_for. Remove their stopped supervisor entries
+        // before deleting the VM workdir.
+        for process in self.supervisor.list().await.unwrap_or_default() {
+            let note =
+                serde_json::from_str::<ProcessAnnotation>(&process.config.note).unwrap_or_default();
+            if note.live_for.as_deref() != Some(id) {
+                continue;
+            }
+            for _ in 0..50 {
+                match self.supervisor.info(&process.config.id).await {
+                    Ok(Some(info)) if info.state.status.is_running() => {
+                        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+                    }
+                    _ => break,
+                }
+            }
+            if let Err(err) = self.supervisor.remove(&process.config.id).await {
+                warn!(
+                    process_id = process.config.id,
+                    "failed to remove dependent process: {err:?}"
+                );
+            }
+        }
+
         // Only delete the workdir for user-initiated removal or if .removing marker exists.
         // Orphaned supervisor processes without the marker keep their data intact.
         let vm_path = self.work_dir(id);
@@ -748,13 +791,14 @@ impl App {
 
         // Clean up orphaned supervisor processes (in supervisor but not loaded as VMs)
         let loaded_vm_ids: HashSet<String> = self.lock().vms.keys().cloned().collect();
-        for (_, process) in &running_vms {
-            if !loaded_vm_ids.contains(&process.config.id) {
+        for (note, process) in &running_vms {
+            let owner = note.live_for.as_deref().unwrap_or(&process.config.id);
+            if !loaded_vm_ids.contains(owner) {
                 info!(
                     "Cleaning up orphaned supervisor process: {}",
                     process.config.id
                 );
-                self.spawn_finish_remove(&process.config.id);
+                self.spawn_finish_remove(owner);
             }
         }
 

dstack/vmm/src/app/qemu.rs

@@ -179,7 +179,8 @@ struct PreparedQemuLaunch {
     hugepage_numa_nodes: Option<HashMap<String, u32>>,
     gpu_numa_nodes: HashMap<String, String>,
     numa_cpus: Option<String>,
-    tpm_path: Option<&'static str>,
+    swtpm_socket: Option<PathBuf>,
+    swtpm_path: Option<PathBuf>,
     tdx_mr_config_id: Option<String>,
     snp_host_data: Option<String>,
     snp_launch_params: Option<AmdSevSnpLaunchParams>,
@@ -219,11 +220,23 @@ impl PreparedQemuLaunch {
         } else {
             None
         };
-        let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) {
-            Some(detect_tpm_device()?)
-        } else {
-            None
-        };
+        let (swtpm_socket, swtpm_path) =
+            if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) {
+                if !vm.manifest.no_tee || !vm.image.info.is_dev {
+                    bail!("swtpm key provider is only available to no-TEE development VMs");
+                }
+                let swtpm_path = which::which("swtpm")
+                    .context("tpm key provider requested but swtpm is not installed")?;
+                let state_dir = workdir.swtpm_state_dir();
+                fs::create_dir_all(&state_dir).context("failed to create swtpm state directory")?;
+                let socket = workdir.swtpm_socket();
+                if socket.exists() {
+                    fs::remove_file(&socket).context("failed to remove stale swtpm socket")?;
+                }
+                (Some(socket), Some(swtpm_path))
+            } else {
+                (None, None)
+            };
         prepare_shared_disk(&workdir, cfg)?;
 
         let tee_enabled = !vm.manifest.no_tee;
@@ -257,7 +270,8 @@ impl PreparedQemuLaunch {
             hugepage_numa_nodes,
             gpu_numa_nodes,
             numa_cpus,
-            tpm_path,
+            swtpm_socket,
+            swtpm_path,
             tdx_mr_config_id,
             snp_host_data,
             snp_launch_params,
@@ -301,16 +315,6 @@ fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> {
     create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk")
 }
 
-fn detect_tpm_device() -> Result<&'static str> {
-    if Path::new("/dev/tpmrm0").exists() {
-        Ok("/dev/tpmrm0")
-    } else if Path::new("/dev/tpm0").exists() {
-        Ok("/dev/tpm0")
-    } else {
-        bail!("tpm key provider requested but no TPM device found on host")
-    }
-}
-
 struct QemuCommandBuilder<'a> {
     vm: &'a VmConfig,
     cfg: &'a CvmConfig,
@@ -326,14 +330,76 @@ impl VmConfig {
         gpus: &GpuConfig,
     ) -> Result<Vec<ProcessConfig>> {
         let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?;
-        let process = QemuCommandBuilder {
+        let mut process = QemuCommandBuilder {
             vm: self,
             cfg,
             gpus,
             prepared: &prepared,
         }
         .build()?;
-        Ok(vec![process])
+        let Some(socket) = prepared.swtpm_socket.as_deref() else {
+            return Ok(vec![process]);
+        };
+        let swtpm_path = prepared
+            .swtpm_path
+            .as_ref()
+            .context("missing swtpm executable for configured socket")?;
+
+        // The supervisor starts process configs in order, but swtpm may need a
+        // moment to create its socket. Make QEMU wait without shell-quoting any
+        // of its arguments: they are passed as positional parameters.
+        let original_command = std::mem::replace(&mut process.command, "/bin/sh".into());
+        let original_args = std::mem::take(&mut process.args);
+        process.args = vec![
+            "-c".into(),
+            "for i in $(seq 1 100); do [ -S \"$1\" ] && shift && exec \"$@\"; sleep 0.05; done; echo 'timed out waiting for swtpm socket' >&2; exit 1".into(),
+            "dstack-qemu-wait-swtpm".into(),
+            socket.to_string_lossy().into_owned(),
+            original_command,
+        ];
+        process.args.extend(original_args);
+
+        let note = serde_json::to_string(&ProcessAnnotation {
+            kind: "swtpm".into(),
+            live_for: Some(self.manifest.id.clone()),
+        })?;
+        let swtpm_args = vec![
+            "socket".into(),
+            "--tpm2".into(),
+            "--tpmstate".into(),
+            format!("dir={}", prepared.workdir.swtpm_state_dir().display()),
+            "--ctrl".into(),
+            format!("type=unixio,path={},mode=0666", socket.display()),
+            "--flags".into(),
+            "not-need-init,startup-clear".into(),
+        ];
+        let swtpm = ProcessConfig {
+            id: format!("{}-swtpm", self.manifest.id),
+            name: format!("{} swtpm", self.manifest.name),
+            command: swtpm_path.to_string_lossy().into_owned(),
+            args: swtpm_args,
+            env: Default::default(),
+            cwd: prepared.workdir.path().to_string_lossy().into_owned(),
+            stdout: prepared
+                .workdir
+                .stdout_file()
+                .to_string_lossy()
+                .into_owned(),
+            stderr: prepared
+                .workdir
+                .stderr_file()
+                .to_string_lossy()
+                .into_owned(),
+            pidfile: prepared
+                .workdir
+                .swtpm_state_dir()
+                .join("supervisor.pid")
+                .to_string_lossy()
+                .into_owned(),
+            cid: None,
+            note,
+        };
+        Ok(vec![swtpm, process])
     }
 }
 
@@ -518,10 +584,12 @@ impl QemuCommandBuilder<'_> {
     }
 
     fn configure_tpm_and_vsock(&self, command: &mut Command) {
-        if let Some(tpm_path) = self.prepared.tpm_path {
+        if let Some(socket) = &self.prepared.swtpm_socket {
             command
+                .arg("-chardev")
+                .arg(format!("socket,id=chrtpm,path={}", socket.display()))
                 .arg("-tpmdev")
-                .arg(format!("passthrough,id=tpm0,path={tpm_path}"))
+                .arg("emulator,id=tpm0,chardev=chrtpm")
                 .arg("-device")
                 .arg("tpm-tis,tpmdev=tpm0");
         }
@@ -970,14 +1038,15 @@ mod tests {
             workdir: PathBuf::from("/does-not-exist/vm-1"),
             gateway_enabled: false,
         };
-        let prepared = PreparedQemuLaunch {
+        let mut prepared = PreparedQemuLaunch {
             workdir: VmWorkDir::new("/does-not-exist/vm-1"),
             platform: TeePlatform::Tdx,
             networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()],
             hugepage_numa_nodes: None,
             gpu_numa_nodes: HashMap::new(),
             numa_cpus: None,
-            tpm_path: None,
+            swtpm_socket: None,
+            swtpm_path: None,
             tdx_mr_config_id: None,
             snp_host_data: None,
             snp_launch_params: None,
@@ -1024,5 +1093,25 @@ mod tests {
             .args
             .iter()
             .any(|arg| arg.contains("virtio-net-pci,netdev=net1")));
+
+        prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock"));
+        let process = QemuCommandBuilder {
+            vm: &vm,
+            cfg: &config.cvm,
+            gpus: &GpuConfig::default(),
+            prepared: &prepared,
+        }
+        .build()
+        .unwrap();
+        assert!(process.args.windows(2).any(|args| {
+            args == [
+                "-chardev",
+                "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock",
+            ]
+        }));
+        assert!(process
+            .args
+            .windows(2)
+            .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"]));
     }
 }

dstack/vmm/src/app/workdir.rs

@@ -104,6 +104,14 @@ impl VmWorkDir {
         self.workdir.join("shared")
     }
 
+    pub fn swtpm_state_dir(&self) -> PathBuf {
+        self.workdir.join("swtpm")
+    }
+
+    pub fn swtpm_socket(&self) -> PathBuf {
+        self.swtpm_state_dir().join("swtpm.sock")
+    }
+
     pub fn app_compose_path(&self) -> PathBuf {
         self.shared_dir().join(APP_COMPOSE)
     }