fix(volume): share duplicate roots across mount targets

Author: kvinwang

docs/verity-volumes.md

@@ -73,6 +73,10 @@ A required volume that is missing, malformed, or fails verification stops guest
 preparation. dm-verity continues verifying blocks lazily as the application
 reads them.
 
+The same `verity_root` may be declared more than once with different targets.
+The VMM attaches one disk for that root, and the guest mounts the verified
+content at each requested target.
+
 For diagnostics, `dstack-volume scan` lists recognized disks and
 `dstack-volume status app-compose.json` compares requested roots with attached
 and active devices.

dstack/crates/dstack-volume/src/bin/dstack-volume.rs

@@ -9,6 +9,7 @@
 //! table. Everything read from a disk is untrusted: kind handlers use the
 //! measured app compose as their source of policy and cryptographic identity.
 
+use std::collections::HashMap;
 use std::io::Read;
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
@@ -96,13 +97,24 @@ fn mount_all(compose_path: PathBuf) -> Result<()> {
         requested = compose.verity_volumes.len(),
         "discovered dstack volumes"
     );
+    let mut active_roots: HashMap<[u8; 32], PathBuf> = HashMap::new();
     for (index, requested) in compose.verity_volumes.iter().enumerate() {
-        activate_requested(index, requested, &volumes).with_context(|| {
+        if let Some(mapped) = active_roots.get(&requested.verity_root) {
+            mount_volume(requested, mapped).with_context(|| {
+                format!(
+                    "failed to mount required volume {index} at {}",
+                    requested.target.display()
+                )
+            })?;
+            continue;
+        }
+        let mapped = activate_requested(index, requested, &volumes).with_context(|| {
             format!(
                 "failed to activate required volume {index} at {}",
                 requested.target.display()
             )
         })?;
+        active_roots.insert(requested.verity_root, mapped);
     }
     Ok(())
 }
@@ -126,10 +138,17 @@ fn status(compose_path: PathBuf) -> Result<()> {
         let attached = volumes
             .iter()
             .any(|volume| volume.root_hash == requested.verity_root);
-        let mapper_name = format!("dstack-verity{index}");
-        let active = Path::new("/dev/mapper").join(&mapper_name).exists()
-            && mapping_root(&mapper_name)?
-                .eq_ignore_ascii_case(&hex::encode(requested.verity_root));
+        let expected_root = hex::encode(requested.verity_root);
+        let mut active = false;
+        for mapper_index in 0..compose.verity_volumes.len() {
+            let mapper_name = format!("dstack-verity{mapper_index}");
+            if Path::new("/dev/mapper").join(&mapper_name).exists()
+                && mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root)
+            {
+                active = true;
+                break;
+            }
+        }
         println!(
             "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}",
             hex::encode(requested.verity_root),
@@ -275,7 +294,7 @@ fn activate_requested(
     index: usize,
     requested: &RequestedVolume,
     volumes: &[VerityVolume],
-) -> Result<()> {
+) -> Result<PathBuf> {
     let candidate = volumes
         .iter()
         .find(|volume| volume.root_hash == requested.verity_root)
@@ -293,7 +312,7 @@ fn activate_requested(
                 return Err(err);
             }
             info!(mapper = mapper_name, "reused active verity mapping");
-            return Ok(());
+            return Ok(mapped);
         }
         run_cmd!(veritysetup close $mapper_name).context("closing stale verity mapping")?;
     }
@@ -308,7 +327,7 @@ fn activate_requested(
         let _ = run_cmd!(veritysetup close $mapper_name);
         return Err(err);
     }
-    Ok(())
+    Ok(mapped)
 }
 
 fn verify_first_block(path: &Path) -> Result<()> {

dstack/dstack-types/src/lib.rs

@@ -141,19 +141,12 @@ pub struct VerityVolume {
     pub target: std::path::PathBuf,
 }
 
-/// Reject ambiguous volume declarations before any disk is attached or
-/// activated. A root identifies one logical volume, and a mount target can
-/// only be owned by one volume.
+/// Reject ambiguous mount declarations before any disk is attached or
+/// activated. The same root may intentionally be mounted at multiple targets,
+/// but a target can only be owned by one volume.
 pub fn validate_verity_volumes(volumes: &[VerityVolume]) -> Result<(), String> {
-    let mut roots = std::collections::HashSet::new();
     let mut targets = std::collections::HashSet::new();
     for volume in volumes {
-        if !roots.insert(volume.verity_root) {
-            return Err(format!(
-                "duplicate verity root {}",
-                hex::encode(volume.verity_root)
-            ));
-        }
         if !targets.insert(&volume.target) {
             return Err(format!(
                 "duplicate verity volume target {}",
@@ -191,10 +184,8 @@ mod verity_volume_tests {
     }
 
     #[test]
-    fn rejects_duplicate_roots_and_targets() {
-        assert!(validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")])
-            .unwrap_err()
-            .contains("duplicate verity root"));
+    fn allows_duplicate_roots_but_rejects_duplicate_targets() {
+        validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]).unwrap();
         assert!(validate_verity_volumes(&[volume(1, "/a"), volume(2, "/a")])
             .unwrap_err()
             .contains("duplicate verity volume target"));

dstack/vmm/src/main_service.rs

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+use std::collections::HashSet;
 use std::ops::Deref;
 use std::path::{Component, Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -237,7 +238,19 @@ fn resolve_volumes(
         bail!("volumes requested but cvm.volumes_dir is not configured");
     }
     let base = fs::canonicalize(dir)?;
+    let mut roots = HashSet::new();
     reqs.iter()
+        .filter(|volume| {
+            let first = roots.insert(volume.verity_root);
+            if !first {
+                warn!(
+                    root = %hex::encode(volume.verity_root),
+                    source = volume.source,
+                    "not attaching duplicate verity root"
+                );
+            }
+            first
+        })
         .map(|v| {
             let real = resolve_volume_source(&base, &v.source)?;
             Ok(crate::app::VmVolume {
@@ -1057,4 +1070,38 @@ mod tests {
         );
         Ok(())
     }
+
+    #[test]
+    fn resolve_volumes_attaches_duplicate_root_once() -> Result<()> {
+        let tmp = tempfile::tempdir()?;
+        fs::write(tmp.path().join("first.img"), b"volume")?;
+        let mut cvm_config = test_cvm_config();
+        cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned();
+        let root = [7; 32];
+
+        let volumes = resolve_volumes(
+            &[
+                dstack_types::VerityVolume {
+                    source: "first.img".into(),
+                    verity_root: root,
+                    target: "/run/first".into(),
+                },
+                dstack_types::VerityVolume {
+                    // This source deliberately does not exist: the first entry
+                    // owns the single attachment for this content root.
+                    source: "duplicate.img".into(),
+                    verity_root: root,
+                    target: "/run/second".into(),
+                },
+            ],
+            &cvm_config,
+        )?;
+
+        assert_eq!(volumes.len(), 1);
+        assert_eq!(
+            volumes[0].source,
+            tmp.path().join("first.img").display().to_string()
+        );
+        Ok(())
+    }
 }