Merge pull request #798 from Dstack-TEE/feat/dev-swtpm

vmm: add per-VM swtpm key provider

Author: kvinwang

CONTRIBUTING.md

@@ -62,6 +62,12 @@ dstack deploy ./docker-compose.yml \
   --no-tee
 ```
 
+To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM
+host and select `key_provider=tpm` in the VMM console (or pass `--key-provider
+tpm` in tooling that exposes the compose option). The VMM keeps the software
+TPM state in the VM work directory so that seal/unseal survives guest restarts.
+The software TPM is host-controlled and does not provide hardware isolation.
+
 The simulator package is installed only in development images. This mode
 provides no hardware isolation and must never be used with production
 workloads or secrets. Real quote generation, hardware isolation, and KMS

docs/security/cvm-boundaries.md

@@ -44,7 +44,7 @@ This is the main configuration file for the application in JSON format:
 | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup |
 | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. |
 | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). |
-| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". `"tpm"` is only supported on platforms whose TPM is part of the platform trust model (GCP vTPM, AWS EC2 NitroTPM); on other platforms guest setup fails closed, since sealing to a host-provided software TPM (e.g. swtpm under bare QEMU) offers no protection against the host. |
+| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. |
 
 
 The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation.

dstack/Cargo.lock

@@ -48,7 +48,7 @@ const AWS_NITRO_PCRS: [u32; 5] = [4, 7, 8, 12, 14];
 
 pub fn dstack_pcr_policy_for_platform(platform: Platform) -> Result<PcrSelection> {
     match platform {
-        Platform::Gcp => Ok(dstack_pcr_policy()),
+        Platform::Dstack | Platform::Gcp => Ok(dstack_pcr_policy()),
         Platform::AwsEc2 => Ok(PcrSelection::sha384(&AWS_NITRO_PCRS)),
         _ => bail!("TPM local key provider is not supported on {platform:?}"),
     }
@@ -367,6 +367,16 @@ mod tests {
         let policy = dstack_pcr_policy_for_platform(Platform::AwsEc2).unwrap();
         assert_eq!(policy.to_arg(), "sha384:4,7,8,12,14");
     }
+
+    #[test]
+    fn dstack_platform_uses_development_tpm_policy() {
+        assert_eq!(
+            dstack_pcr_policy_for_platform(Platform::Dstack)
+                .unwrap()
+                .to_arg(),
+            "sha256:0,2,14"
+        );
+    }
 }
 
 mod gcp_ak;

dstack/tpm-attest/src/lib.rs

@@ -15,6 +15,7 @@ rocket-vsock-listener = { workspace = true }
 tracing.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 anyhow.workspace = true
+libc.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 shared_child.workspace = true
@@ -63,6 +64,7 @@ tar.workspace = true
 
 [dev-dependencies]
 insta.workspace = true
+tempfile.workspace = true
 
 [build-dependencies]
 or-panic.workspace = true

dstack/vmm/Cargo.toml

@@ -47,6 +47,30 @@ pub(crate) mod registry;
 mod vm_info;
 mod workdir;
 
+fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> {
+    // SAFETY: pidfd syscalls receive scalar arguments and a null siginfo pointer.
+    let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) };
+    if fd < 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    let result = unsafe {
+        libc::syscall(
+            libc::SYS_pidfd_send_signal,
+            fd,
+            signal,
+            std::ptr::null::<libc::siginfo_t>(),
+            0,
+        )
+    };
+    // SAFETY: fd was returned by pidfd_open and is owned by this function.
+    unsafe { libc::close(fd as libc::c_int) };
+    if result < 0 {
+        Err(std::io::Error::last_os_error())
+    } else {
+        Ok(())
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct PortMapping {
     pub address: IpAddr,
@@ -425,10 +449,44 @@ impl App {
 
     pub async fn stop_vm(&self, id: &str) -> Result<()> {
         self.set_started(id, false)?;
-        self.supervisor.stop(id).await?;
+        self.stop_vm_process(id).await?;
         Ok(())
     }
 
+    async fn stop_vm_process(&self, id: &str) -> Result<()> {
+        let Some(info) = self.supervisor.info(id).await? else {
+            return Ok(());
+        };
+        // Non-TPM VMs run QEMU directly and keep the existing Supervisor stop
+        // path. Only the TPM launcher's hidden subcommand implements graceful
+        // child-process shutdown.
+        if info.config.args.first().map(String::as_str) != Some("vm-launcher") {
+            return self.supervisor.stop(id).await;
+        }
+        if info.state.status.is_running() {
+            let pid = info.state.pid.context("running VM launcher has no PID")?;
+            if let Err(error) = signal_pidfd(pid, libc::SIGTERM) {
+                warn!(id, %pid, %error, "failed to signal VM launcher gracefully; forcing shutdown");
+                return self.supervisor.stop(id).await;
+            }
+            for _ in 0..150 {
+                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+                let running = self
+                    .supervisor
+                    .info(id)
+                    .await?
+                    .is_some_and(|info| info.state.status.is_running());
+                if !running {
+                    // Synchronize Supervisor's `started` flag after the launcher
+                    // completed its graceful child cleanup.
+                    return self.supervisor.stop(id).await;
+                }
+            }
+            warn!(id, "VM launcher did not stop gracefully; forcing shutdown");
+        }
+        self.supervisor.stop(id).await
+    }
+
     pub async fn remove_vm(&self, id: &str) -> Result<()> {
         {
             let mut state = self.lock();
@@ -464,8 +522,8 @@ impl App {
     /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup.
     async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> {
         // Stop the supervisor process (idempotent if already stopped)
-        if let Err(err) = self.supervisor.stop(id).await {
-            debug!("supervisor.stop({id}) during removal: {err:?}");
+        if let Err(err) = self.stop_vm_process(id).await {
+            debug!("graceful VM stop during removal failed: {err:?}");
         }
 
         // Poll until the process is no longer running, then remove it.

dstack/vmm/src/app.rs

@@ -6,6 +6,7 @@
 use crate::{
     app::Manifest,
     config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform},
+    vm_launcher::{ChildCommand, LaunchSpec},
 };
 use std::collections::HashMap;
 use std::os::unix::fs::PermissionsExt;
@@ -29,6 +30,7 @@ use anyhow::{bail, Context, Result};
 use bon::Builder;
 use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind};
 use fs_err as fs;
+use nix::unistd::User;
 use serde::Serialize;
 use supervisor_client::supervisor::ProcessConfig;
 
@@ -179,7 +181,8 @@ struct PreparedQemuLaunch {
     hugepage_numa_nodes: Option<HashMap<String, u32>>,
     gpu_numa_nodes: HashMap<String, String>,
     numa_cpus: Option<String>,
-    tpm_path: Option<&'static str>,
+    swtpm_socket: Option<PathBuf>,
+    swtpm_path: Option<PathBuf>,
     tdx_mr_config_id: Option<String>,
     snp_host_data: Option<String>,
     snp_launch_params: Option<AmdSevSnpLaunchParams>,
@@ -219,11 +222,20 @@ impl PreparedQemuLaunch {
         } else {
             None
         };
-        let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) {
-            Some(detect_tpm_device()?)
-        } else {
-            None
-        };
+        let (swtpm_socket, swtpm_path) =
+            if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) {
+                let swtpm_path = which::which("swtpm")
+                    .context("tpm key provider requested but swtpm is not installed")?;
+                let state_dir = workdir.swtpm_state_dir();
+                fs::create_dir_all(&state_dir).context("failed to create swtpm state directory")?;
+                let socket = workdir.swtpm_socket();
+                if socket.exists() {
+                    fs::remove_file(&socket).context("failed to remove stale swtpm socket")?;
+                }
+                (Some(socket), Some(swtpm_path))
+            } else {
+                (None, None)
+            };
         prepare_shared_disk(&workdir, cfg)?;
 
         let tee_enabled = !vm.manifest.no_tee;
@@ -257,7 +269,8 @@ impl PreparedQemuLaunch {
             hugepage_numa_nodes,
             gpu_numa_nodes,
             numa_cpus,
-            tpm_path,
+            swtpm_socket,
+            swtpm_path,
             tdx_mr_config_id,
             snp_host_data,
             snp_launch_params,
@@ -301,16 +314,6 @@ fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> {
     create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk")
 }
 
-fn detect_tpm_device() -> Result<&'static str> {
-    if Path::new("/dev/tpmrm0").exists() {
-        Ok("/dev/tpmrm0")
-    } else if Path::new("/dev/tpm0").exists() {
-        Ok("/dev/tpm0")
-    } else {
-        bail!("tpm key provider requested but no TPM device found on host")
-    }
-}
-
 struct QemuCommandBuilder<'a> {
     vm: &'a VmConfig,
     cfg: &'a CvmConfig,
@@ -333,7 +336,71 @@ impl VmConfig {
             prepared: &prepared,
         }
         .build()?;
-        Ok(vec![process])
+        let Some(socket) = prepared.swtpm_socket.as_deref() else {
+            return Ok(vec![process]);
+        };
+        let swtpm_path = prepared
+            .swtpm_path
+            .as_ref()
+            .context("missing swtpm executable for configured socket")?;
+        let (socket_uid, socket_gid) = if cfg.user.is_empty() {
+            (unsafe { libc::geteuid() }, unsafe { libc::getegid() })
+        } else {
+            let user = User::from_name(&cfg.user)
+                .context("failed to resolve QEMU user")?
+                .with_context(|| format!("QEMU user {} does not exist", cfg.user))?;
+            (user.uid.as_raw(), user.gid.as_raw())
+        };
+
+        let swtpm_args = vec![
+            "socket".into(),
+            "--tpm2".into(),
+            "--tpmstate".into(),
+            format!("dir={}", prepared.workdir.swtpm_state_dir().display()),
+            "--ctrl".into(),
+            format!(
+                "type=unixio,path={},mode=0600,uid={socket_uid},gid={socket_gid}",
+                socket.display()
+            ),
+            "--flags".into(),
+            "not-need-init,startup-clear".into(),
+        ];
+        let spec = LaunchSpec {
+            qemu: ChildCommand {
+                command: process.command,
+                args: process.args,
+            },
+            swtpm: ChildCommand {
+                command: swtpm_path.to_string_lossy().into_owned(),
+                args: swtpm_args,
+            },
+            swtpm_socket: socket.to_path_buf(),
+            startup_timeout_ms: 5_000,
+            shutdown_timeout_ms: 10_000,
+        };
+        let spec_path = prepared.workdir.launch_spec_path();
+        safe_write::safe_write(&spec_path, serde_json::to_vec_pretty(&spec)?)
+            .context("failed to write VM launch specification")?;
+        let executable =
+            std::env::current_exe().context("failed to locate dstack-vmm executable")?;
+        let launcher = ProcessConfig {
+            id: self.manifest.id.clone(),
+            name: self.manifest.name.clone(),
+            command: executable.to_string_lossy().into_owned(),
+            args: vec![
+                "vm-launcher".into(),
+                "--spec".into(),
+                spec_path.to_string_lossy().into_owned(),
+            ],
+            env: process.env,
+            cwd: process.cwd,
+            stdout: process.stdout,
+            stderr: process.stderr,
+            pidfile: process.pidfile,
+            cid: process.cid,
+            note: process.note,
+        };
+        Ok(vec![launcher])
     }
 }
 
@@ -518,10 +585,12 @@ impl QemuCommandBuilder<'_> {
     }
 
     fn configure_tpm_and_vsock(&self, command: &mut Command) {
-        if let Some(tpm_path) = self.prepared.tpm_path {
+        if let Some(socket) = &self.prepared.swtpm_socket {
             command
+                .arg("-chardev")
+                .arg(format!("socket,id=chrtpm,path={}", socket.display()))
                 .arg("-tpmdev")
-                .arg(format!("passthrough,id=tpm0,path={tpm_path}"))
+                .arg("emulator,id=tpm0,chardev=chrtpm")
                 .arg("-device")
                 .arg("tpm-tis,tpmdev=tpm0");
         }
@@ -970,14 +1039,15 @@ mod tests {
             workdir: PathBuf::from("/does-not-exist/vm-1"),
             gateway_enabled: false,
         };
-        let prepared = PreparedQemuLaunch {
+        let mut prepared = PreparedQemuLaunch {
             workdir: VmWorkDir::new("/does-not-exist/vm-1"),
             platform: TeePlatform::Tdx,
             networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()],
             hugepage_numa_nodes: None,
             gpu_numa_nodes: HashMap::new(),
             numa_cpus: None,
-            tpm_path: None,
+            swtpm_socket: None,
+            swtpm_path: None,
             tdx_mr_config_id: None,
             snp_host_data: None,
             snp_launch_params: None,
@@ -1024,5 +1094,25 @@ mod tests {
             .args
             .iter()
             .any(|arg| arg.contains("virtio-net-pci,netdev=net1")));
+
+        prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock"));
+        let process = QemuCommandBuilder {
+            vm: &vm,
+            cfg: &config.cvm,
+            gpus: &GpuConfig::default(),
+            prepared: &prepared,
+        }
+        .build()
+        .unwrap();
+        assert!(process.args.windows(2).any(|args| {
+            args == [
+                "-chardev",
+                "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock",
+            ]
+        }));
+        assert!(process
+            .args
+            .windows(2)
+            .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"]));
     }
 }

dstack/vmm/src/app/qemu.rs

@@ -104,6 +104,18 @@ impl VmWorkDir {
         self.workdir.join("shared")
     }
 
+    pub fn swtpm_state_dir(&self) -> PathBuf {
+        self.workdir.join("swtpm")
+    }
+
+    pub fn swtpm_socket(&self) -> PathBuf {
+        self.swtpm_state_dir().join("swtpm.sock")
+    }
+
+    pub fn launch_spec_path(&self) -> PathBuf {
+        self.workdir.join("launch.json")
+    }
+
     pub fn app_compose_path(&self) -> PathBuf {
         self.shared_dir().join(APP_COMPOSE)
     }